diff --git a/app/all-chats/page.tsx b/app/all-chats/page.tsx index d1881d0..3ada7ad 100644 --- a/app/all-chats/page.tsx +++ b/app/all-chats/page.tsx @@ -7,7 +7,7 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" import { Input } from "@/components/ui/input" import { Brain, ArrowLeft, MessageSquare, Search, Plus, Trash2, MoreHorizontal, Star, AlertTriangle } from "lucide-react" import { SparklesCore } from "@/components/sparkles" -import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu" +import ChatOptionsCard from "@/components/chat/ChatOptionsCard" import { chatService } from "@/lib/chat-service" // Error component @@ -40,6 +40,18 @@ export default function AllChatsPage() { const [error, setError] = useState(null) const [chatHistory, setChatHistory] = useState([]) + // Placeholder for pinning/unpinning a chat + const handlePinToggle = (chatId: string, pinned: boolean) => { + console.log(`Attempting to ${pinned ? 'pin' : 'unpin'} chat ${chatId}`); + // TODO: Implement actual pin/unpin logic here + }; + + // Placeholder for deleting a chat + const handleDeleteChat = (chatId: string) => { + console.log(`Attempting to delete chat ${chatId}`); + // TODO: Implement actual delete logic here + }; + // Fetch chats from the database useEffect(() => { const fetchChats = async () => { @@ -71,6 +83,14 @@ export default function AllChatsPage() { return } + if (isLoading) { + return ( +
+ Loading chats... +
+ ); + } + return (
{/* Interactive background with moving particles */} @@ -134,7 +154,12 @@ export default function AllChatsPage() {
{pinnedChats.map((chat) => ( - + ))}
@@ -145,7 +170,12 @@ export default function AllChatsPage() {

Recent Chats

{unpinnedChats.length > 0 ? ( - unpinnedChats.map((chat) => ) + unpinnedChats.map((chat) => ) ) : (

{isLoading @@ -164,7 +194,7 @@ export default function AllChatsPage() { ) } -function ChatCard({ chat }: { chat: any }) { +function ChatCard({ chat, onPinToggle, onDelete }: { chat: any; onPinToggle: (chatId: string, pinned: boolean) => void; onDelete: (chatId: string) => void }) { return ( @@ -180,23 +210,7 @@ function ChatCard({ chat }: { chat: any }) { )}

- - - - - - - - {chat.pinned ? "Unpin" : "Pin"} - - - - Delete - - - +

diff --git a/app/ar-mode/ar-mode-content.tsx b/app/ar-mode/ar-mode-content.tsx new file mode 100644 index 0000000..79192e8 --- /dev/null +++ b/app/ar-mode/ar-mode-content.tsx @@ -0,0 +1,1376 @@ +"use client" + +import React, { useState, useEffect, useRef, useCallback } from "react" +import { useRouter } from "next/navigation" +import { Button } from "@/components/ui/button" +import { ArrowLeft, Camera, Mic, Loader2, AlertTriangle, Volume2, X, RefreshCw, CircleDot, ArrowRight } from "lucide-react" +import { motion, AnimatePresence } from "framer-motion" +import ReactMarkdown from "react-markdown" +import { useSwipeable } from 'react-swipeable' +import { getGroqVisionAnalysis, getGroqTranscription } from "@/lib/groq-service" +import { speakText } from "@/lib/tts-service" +import InfoWidget from '@/components/ar/InfoWidget' + +// THESE IMPORTS ARE LIKELY NEEDED BASED ON LINTER ERRORS - ADDING THEM HERE +import * as faceapi from 'face-api.js'; +import { FilesetResolver, ObjectDetector, GestureRecognizer, GestureRecognizerResult } from "@mediapipe/tasks-vision"; + + +// Placeholder for DETAILED_BASE_PROMPT if not defined elsewhere +const DETAILED_BASE_PROMPT = "You are a compassionate and intuitive vision assistant AI, designed to help individuals understand their surroundings through detailed descriptions. Your task is to guide me, as I am visually impaired, in perceiving the environment around me by providing insights about people, locations, objects, and more. \n\nPlease describe the scene around me, focusing on the following elements: \n\nPeople: __________ \nLocation: __________ \nObjects: __________ \nActivities: __________\n\n\nOutput Format:Begin your response with the phrase \"I see...\" followed by a vivid description of the surroundings. Use bold text to emphasize important details or actions. \n\nDetails: \n\nProvide sensory details that evoke a sense of the environment, including sounds, smells, temperatures, and textures. \nBe empathetic and supportive, as if you are my companion, ensuring that your tone is warm and encouraging. \nConsider the context of the location and the activities happening around me.\n\n\nExamples: \n\n\"I see a group of people laughing and talking near a vibrant café, the aroma of freshly brewed coffee fills the air.\" \n\"I see a park with children playing; the sound of laughter mixes with the rustling of leaves in the gentle breeze.\"\n\n\nConstraints: \n\nAvoid using overly technical language; keep the descriptions clear and relatable. \nEnsure that the descriptions are not overwhelming; focus on providing a balanced overview of the surroundings. \nDo not assume prior knowledge; provide context where necessary for clarity."; + + +export default function ARModeContent() { // Renamed from ARModePage + const router = useRouter() + const videoRef = useRef(null) + const canvasRef = useRef(null) + const [stream, setStream] = useState(null) + const [error, setError] = useState(null) + const [isLoading, setIsLoading] = useState(false) + const [aiResponse, setAiResponse] = useState(null) + const [isAnalyzing, setIsAnalyzing] = useState(false) + const [userQuery, setUserQuery] = useState("Describe what you see in detail.") + const [isMicListening, setIsMicListening] = useState(false) + const [isTranscribing, setIsTranscribing] = useState(false) + const [isSpeaking, setIsSpeaking] = useState(false) + const [statusMessage, setStatusMessage] = useState(null) + const [isFrontCamera, setIsFrontCamera] = useState(false) + const mediaRecorderRef = useRef(null) + const audioChunksRef = useRef([]) + + // State variables that were causing "Cannot find name" errors + const [modelsReady, setModelsReady] = useState(false); + const [mediaPipeModelsReady, setMediaPipeModelsReady] = useState(false); + const [faceApiError, setFaceApiError] = useState(null); + const [mediaPipeError, setMediaPipeError] = useState(null); + const [objectDetector, setObjectDetectorState] = useState(null); // Corrected state name + const [gestureRecognizer, setGestureRecognizerState] = useState(null); // Corrected state name + const lastVideoTimeRef = useRef(0); + const [identityInfoContent, setIdentityInfoContent] = useState(null); + const [objectCardContent, setObjectCardContent] = useState(null); + const [capturedImagePreviewUrl, setCapturedImagePreviewUrl] = useState(null); + const [showCards, setShowCards] = useState(false); + const [lastAnalysisTime, setLastAnalysisTime] = useState(null); + const [recordingTime, setRecordingTime] = useState(0); + const recordingIntervalRef = useRef(null); + const captureIntervalRef = useRef(null); + const [recordStartTime, setRecordStartTime] = useState(null); + const [isRecording, setIsRecording] = useState(false); + const [activeCardIndex, setActiveCardIndex] = useState(0); + const [isMobile, setIsMobile] = useState(false); + + + // Face-API model loading options + const getTinyFaceDetectorOptions = useCallback(() => { + return new faceapi.TinyFaceDetectorOptions({ inputSize: 320, scoreThreshold: 0.5 }); + }, []); + + // Function to load FaceAPI models + const loadModels = useCallback(async () => { + const MODEL_URL = '/models/faceapi'; // Ensure this path is correct + try { + console.log("Loading Face-API models..."); + await Promise.all([ + faceapi.nets.tinyFaceDetector.loadFromUri(MODEL_URL), + faceapi.nets.faceLandmark68Net.loadFromUri(MODEL_URL), + faceapi.nets.faceRecognitionNet.loadFromUri(MODEL_URL), + faceapi.nets.faceExpressionNet.loadFromUri(MODEL_URL), + faceapi.nets.ageGenderNet.loadFromUri(MODEL_URL) + ]); + setModelsReady(true); + console.log("Face-API models loaded successfully."); + setFaceApiError(null); + } catch (err) { + console.error("Error loading Face-API models:", err); + const message = err instanceof Error ? err.message : "Unknown error loading face models"; + setFaceApiError(`Failed to load face models: ${message}. Ensure model files are accessible at ${MODEL_URL}.`); + setModelsReady(false); + } + }, [setModelsReady, setFaceApiError]); + + const areModelsLoaded = useCallback(() => { // Basic check, relies on modelsReady state + return modelsReady; + }, [modelsReady]); + + // Load FaceAPI models on mount + useEffect(() => { + loadModels(); + }, [loadModels]); + + // Load MediaPipe Models on mount + useEffect(() => { + const initializeMediaPipe = async () => { + try { + console.log("Initializing MediaPipe Vision Tasks..."); + const vision = await FilesetResolver.forVisionTasks( + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" + ); + + // Object Detector + const newObjectDetector = await ObjectDetector.createFromOptions(vision, { + baseOptions: { modelAssetPath: `/mediapipe-models/efficientdet_lite0.tflite`, delegate: "GPU" }, + scoreThreshold: 0.5, + runningMode: "VIDEO", + }); + setObjectDetectorState(newObjectDetector); + console.log("MediaPipe Object Detector created."); + + // Gesture Recognizer + const newGestureRecognizer = await GestureRecognizer.createFromOptions(vision, { + baseOptions: { modelAssetPath: `/mediapipe-models/gesture_recognizer.task`, delegate: "GPU" }, + runningMode: "VIDEO", + numHands: 2, + }); + setGestureRecognizerState(newGestureRecognizer); + console.log("MediaPipe Gesture Recognizer created."); + + setMediaPipeModelsReady(true); + setMediaPipeError(null); + console.log("MediaPipe models initialized successfully."); + } catch (err) { + console.error("Error initializing MediaPipe models:", err); + const message = err instanceof Error ? err.message : "Unknown error loading MediaPipe models"; + setMediaPipeError(`Failed to load MediaPipe models: ${message}.`); + setMediaPipeModelsReady(false); + } + }; + initializeMediaPipe(); + }, [setMediaPipeModelsReady, setMediaPipeError, setObjectDetectorState, setGestureRecognizerState]); + + + // Function to setup camera + const setupCamera = useCallback(async () => { + setError(null) + // Stop previous stream if it exists (important when flipping) + setStream(currentStream => { + if (currentStream) { + console.log("Stopping previous camera stream...") + currentStream.getTracks().forEach((track) => track.stop()) + } + return null; + }); + console.log(`Attempting to access ${isFrontCamera ? 'front' : 'rear'} camera...`) + try { + const mediaStream = await navigator.mediaDevices.getUserMedia({ + video: { facingMode: isFrontCamera ? "user" : "environment" }, + audio: false, + }) + setStream(mediaStream) + if (videoRef.current) { + videoRef.current.srcObject = mediaStream + } + } catch (err) { + console.error("Error accessing camera:", err) + if (err instanceof Error) { + if (err.name === "NotAllowedError") { + setError("Camera permission denied. Please enable camera access in your browser settings.") + } else if (err.name === "NotFoundError" || err.name === "DevicesNotFoundError") { + setError("No suitable camera found. Please ensure a camera is connected and enabled."); + } else { + setError(`Error accessing camera: ${err.message}`) + } + } else { + setError("An unknown error occurred while accessing the camera.") + } + } + }, [isFrontCamera]) + + // Setup camera on mount and when isFrontCamera changes + useEffect(() => { + setupCamera(); + // Cleanup stream on unmount + return () => { + setStream(currentStream => { + if (currentStream) { + console.log("Cleaning up camera stream...") + currentStream.getTracks().forEach((track) => track.stop()) + } + return null; // Ensure stream state is reset + }); + } + }, [setupCamera]); + + // Function to capture frame + const captureFrame = useCallback((): string | null => { + if (!videoRef.current || !canvasRef.current) { + console.error("Video or canvas ref not available") + return null + } + const video = videoRef.current + const canvas = canvasRef.current + canvas.width = video.videoWidth + canvas.height = video.videoHeight + const context = canvas.getContext("2d") + if (!context) { + console.error("Could not get canvas context") + return null + } + context.drawImage(video, 0, 0, canvas.width, canvas.height) + // Convert to base64 JPEG + try { + return canvas.toDataURL("image/jpeg", 0.9) // Adjust quality as needed + } catch (e) { + console.error("Error converting canvas to data URL:", e) + setError("Failed to capture frame.") + return null + } + }, []) + + // Helper function for Face-API.js analysis - now for one-time analysis on a given media element + const performFaceApiAnalysis = useCallback(async (mediaElement: HTMLVideoElement | HTMLImageElement | HTMLCanvasElement | null) => { + if (!modelsReady || !mediaElement) { + console.warn("Face-API models not ready or media element not available for analysis."); + setIdentityInfoContent(prev => prev === null ? `**Identity:**\n- Models not ready` : prev); + return null; + } + + // Check if it's a video element and if it's ready + if (mediaElement instanceof HTMLVideoElement && (mediaElement.readyState < mediaElement.HAVE_METADATA || mediaElement.paused || mediaElement.ended)) { + console.warn("Video element not ready, paused, or ended for Face-API analysis."); + setIdentityInfoContent(prev => prev === null ? `**Identity:**\n- Video not ready` : prev); + return null; + } + + setFaceApiError(null); + + try { + console.log("Performing face-api.js detection..."); + const detections = await faceapi + .detectAllFaces(mediaElement, getTinyFaceDetectorOptions()) + .withFaceLandmarks() + .withFaceExpressions() + .withAgeAndGender(); + + if (detections && detections.length > 0) { + const firstDetection = detections[0]; + const { age, gender, expressions } = firstDetection; + + let dominantExpression = "neutral"; + let maxProbability = 0; + + if (expressions && typeof expressions === 'object') { + for (const expressionKey in expressions) { + if (Object.prototype.hasOwnProperty.call(expressions, expressionKey)) { + const probability = (expressions as any)[expressionKey]; + if (typeof probability === 'number' && probability > maxProbability) { + maxProbability = probability; + dominantExpression = expressionKey; + } + } + } + } + + const identityText = `🎂 Age: ~${Math.round(age)} years\n👤 Gender: ${gender.charAt(0).toUpperCase() + gender.slice(1)}\n😀 Mood: ${dominantExpression.charAt(0).toUpperCase() + dominantExpression.slice(1)}`; + setIdentityInfoContent(identityText); + console.log("One-time Face-API Analysis Result:", { age: Math.round(age), gender, mood: dominantExpression }); + return { age: Math.round(age), gender, mood: dominantExpression }; + } else { + // Simplified message for no face detection + const blurryMessage = "No Clear Face Detected Check the Lighting"; + setIdentityInfoContent(blurryMessage); + console.log("No human faces detected by Face-API in the captured image."); + return null; + } + } catch (err) { + console.error("Error during one-time face-api.js detection:", err); + const errorMessage = err instanceof Error ? err.message : "Face detection failed on captured image"; + + // Specific error handling message + const errorText = `**Detection Error:**\n🤔 ${errorMessage}\n🤔 Try adjusting camera\n🤔 Ensure good lighting`; + + setFaceApiError(errorMessage); + setIdentityInfoContent(errorText); + return null; + } + }, [modelsReady, getTinyFaceDetectorOptions, setIdentityInfoContent, setFaceApiError]); + + // --- New function for MediaPipe Analysis --- + const performMediaPipeAnalysis = useCallback(async (videoElement: HTMLVideoElement | null): Promise<{ objectsText: string | null; gesturesText: string | null }> => { + if (!mediaPipeModelsReady || !objectDetector || !gestureRecognizer || !videoElement) { + console.warn("MediaPipe models not ready or video element not available for analysis."); + return { objectsText: `**Objects/Gestures:**\n- MediaPipe not ready`, gesturesText: null }; + } + + if (videoElement.readyState < videoElement.HAVE_METADATA || videoElement.paused || videoElement.ended) { + console.warn("Video element not ready, paused, or ended for MediaPipe analysis."); + return { objectsText: `**Objects/Gestures:**\n- Video not ready`, gesturesText: null }; + } + + lastVideoTimeRef.current = videoElement.currentTime; + + + let objectsText: string | null = null; + let gesturesText: string | null = null; + + try { + // Object Detection + const objectDetections = objectDetector.detectForVideo(videoElement, performance.now()); + if (objectDetections && objectDetections.detections.length > 0) { + const detectedObjectNames = objectDetections.detections + .map(det => det.categories[0]?.categoryName || "Unknown Object") + .filter((name, index, self) => self.indexOf(name) === index); // Unique names + + if (detectedObjectNames.length > 0) { + objectsText = `**Detected Objects:**\n${detectedObjectNames.map(name => `📦 ${name}`).join('\n')}`; + } else { + objectsText = "**Detected Objects:**\n- None clearly identified"; + } + console.log("MediaPipe Object Detection Result:", detectedObjectNames); + } else { + objectsText = "**Detected Objects:**\n- None"; + console.log("No objects detected by MediaPipe."); + } + + // Gesture Recognition + const gestureRecognitionResult: GestureRecognizerResult = gestureRecognizer.recognizeForVideo(videoElement, performance.now()); + if (gestureRecognitionResult.gestures.length > 0) { + const gestureMap: { [key: string]: string } = { + "None": "None", + "Closed_Fist": "Closed Fist ✊", + "Open_Palm": "Open Palm ðŸ–ï¸", + "Pointing_Up": "Pointing Up â˜ï¸", + "Thumb_Down": "Thumbs Down 👎", + "Thumb_Up": "Thumbs Up ðŸ‘", + "Victory": "Victory ✌ï¸", + "ILoveYou": "I Love You 🤟", + "Unknown": "Unknown Gesture â”" + }; + + const recognizedGestures = gestureRecognitionResult.gestures + .map(gestureData => { + const categoryName = gestureData[0]?.categoryName || "Unknown"; + return gestureMap[categoryName] || `${categoryName} (Unknown)`; // Fallback for unmapped gestures + }) + .filter((name, index, self) => self.indexOf(name) === index && name !== "None"); // Unique and not "None" + + if (recognizedGestures.length > 0) { + gesturesText = `**Detected Gestures:**\n${recognizedGestures.map(gesture => `🤚 ${gesture}`).join('\n')}`; + } else { + gesturesText = "**Detected Gestures:**\n- None clearly identified"; + } + console.log("MediaPipe Gesture Recognition Result:", recognizedGestures); + } else { + gesturesText = "**Detected Gestures:**\n- None"; + console.log("No gestures detected by MediaPipe."); + } + + } catch (err) { + console.error("Error during MediaPipe analysis:", err); + const mediaPipeErrorMessage = err instanceof Error ? err.message : "MediaPipe analysis failed"; + setMediaPipeError(mediaPipeErrorMessage); // Set specific MediaPipe error + // Optionally, update objectsText/gesturesText to show error + objectsText = `**Object Detection Error:**\n- ${mediaPipeErrorMessage}`; + gesturesText = `**Gesture Recognition Error:**\n- ${mediaPipeErrorMessage}`; + } + + return { objectsText, gesturesText }; + }, [mediaPipeModelsReady, objectDetector, gestureRecognizer, setMediaPipeError, lastVideoTimeRef]); + + // Format recording time as MM:SS + const formatRecordingTime = (seconds: number): string => { + const mins = Math.floor(seconds / 60).toString().padStart(2, '0'); + const secs = (seconds % 60).toString().padStart(2, '0'); + return `${mins}:${secs}`; + }; + + // Capture and analyze a single frame + const captureAndAnalyzeFrame = useCallback(async () => { + console.log("[captureAndAnalyzeFrame] Starting frame analysis (called from interval or initial trigger)."); + + const imageDataUrl = captureFrame(); + if (!imageDataUrl) { + console.error("[captureAndAnalyzeFrame] Could not capture frame."); + return; + } + + setCapturedImagePreviewUrl(imageDataUrl); + console.log("[captureAndAnalyzeFrame] Captured image preview URL set."); + + try { + await performFaceApiAnalysis(videoRef.current); + console.log("[captureAndAnalyzeFrame] Face analysis complete."); + } catch (err) { + console.error("[captureAndAnalyzeFrame] Face analysis error:", err); + } + + try { + const { objectsText, gesturesText } = await performMediaPipeAnalysis(videoRef.current); + let combinedObjectGestureContent = ""; + if (objectsText) combinedObjectGestureContent += objectsText; + if (gesturesText) { + if (combinedObjectGestureContent) combinedObjectGestureContent += "\n\n"; + combinedObjectGestureContent += gesturesText; + } + if (!combinedObjectGestureContent) { + combinedObjectGestureContent = "**Objects & Gestures:**\n- None detected or analysis issue."; + } + setObjectCardContent(combinedObjectGestureContent); + console.log("[captureAndAnalyzeFrame] MediaPipe analysis complete. Content:", combinedObjectGestureContent); + } catch (err) { + console.error("[captureAndAnalyzeFrame] MediaPipe analysis error:", err); + } + + try { + const response = await getGroqVisionAnalysis(imageDataUrl, DETAILED_BASE_PROMPT); + setAiResponse(response); + console.log("[captureAndAnalyzeFrame] Groq vision analysis complete. Response:", response); + } catch (err) { + console.error("[captureAndAnalyzeFrame] Groq vision analysis error:", err); + } + + setShowCards(true); + setLastAnalysisTime(Date.now()); + console.log("[captureAndAnalyzeFrame] setShowCards(true) called. lastAnalysisTime set."); + }, [ + captureFrame, + DETAILED_BASE_PROMPT, + performFaceApiAnalysis, + performMediaPipeAnalysis, + getGroqVisionAnalysis, + setCapturedImagePreviewUrl, + setIdentityInfoContent, + setObjectCardContent, + setAiResponse, + setShowCards, + setLastAnalysisTime, + videoRef // videoRef.current is used by analysis helpers + ]); + + // Effect to handle recording logic (timer and periodic capture) + useEffect(() => { + let localRecordingTimerId: NodeJS.Timeout | null = null; + let localFrameAnalysisIntervalId: NodeJS.Timeout | null = null; + + if (isRecording) { + console.log("[Recording Effect] Recording started. Performing initial capture."); + setStatusMessage("Smart recording initializing..."); + + captureAndAnalyzeFrame().then(() => { + setStatusMessage("Smart recording..."); + }); + + localRecordingTimerId = setInterval(() => { + setRecordingTime(prev => prev + 1); + }, 1000); + recordingIntervalRef.current = localRecordingTimerId; + + localFrameAnalysisIntervalId = setInterval(() => { + console.log("[Recording Effect] Interval: Capturing and analyzing frame."); + captureAndAnalyzeFrame(); + }, 4000); + captureIntervalRef.current = localFrameAnalysisIntervalId; + + setRecordStartTime(Date.now()); + + } else { + console.log("[Recording Effect] Recording stopped."); + if (recordingIntervalRef.current) { + clearInterval(recordingIntervalRef.current); + recordingIntervalRef.current = null; + } + if (captureIntervalRef.current) { + clearInterval(captureIntervalRef.current); + captureIntervalRef.current = null; + } + setRecordingTime(0); + setRecordStartTime(null); + if (statusMessage === "Smart recording..." || statusMessage === "Smart recording initializing...") { + setStatusMessage(null); + } + } + + return () => { + console.log("[Recording Effect] Cleanup: Clearing intervals from effect."); + if (localRecordingTimerId) clearInterval(localRecordingTimerId); + if (localFrameAnalysisIntervalId) clearInterval(localFrameAnalysisIntervalId); + + if (recordingIntervalRef.current === localRecordingTimerId) recordingIntervalRef.current = null; + if (captureIntervalRef.current === localFrameAnalysisIntervalId) captureIntervalRef.current = null; + }; + }, [isRecording, captureAndAnalyzeFrame, setStatusMessage, setRecordingTime, setRecordStartTime]); + + // Handle Record Toggle + const handleRecordToggle = useCallback(() => { + if (!modelsReady || !mediaPipeModelsReady) { + setError("AI models are not ready yet. Please wait or refresh."); + return; + } + + setIsRecording(prevIsRecording => { + const newIsRecording = !prevIsRecording; + if (newIsRecording) { + console.log("[handleRecordToggle] Starting recording process..."); + setCapturedImagePreviewUrl(null); + setObjectCardContent(null); + setIdentityInfoContent(null); + setAiResponse(null); + setShowCards(false); + setLastAnalysisTime(null); + } else { + console.log("[handleRecordToggle] Stopping recording process..."); + } + return newIsRecording; + }); + }, [modelsReady, mediaPipeModelsReady, setIsRecording, setCapturedImagePreviewUrl, setObjectCardContent, setIdentityInfoContent, setAiResponse, setShowCards, setLastAnalysisTime, setError]); + + // Handle Analyze Image (one-time capture) + const handleAnalyzeImage = useCallback(async () => { + setStatusMessage("Capturing image...") + setIsAnalyzing(true) + setAiResponse(null) + setError(null) + // Ensure all cards are hidden initially for new analysis + setCapturedImagePreviewUrl(null); + setObjectCardContent(null); + setIdentityInfoContent(null); + setShowCards(false); + setFaceApiError(null); + setMediaPipeError(null); + + const imageDataUrl = captureFrame() + + if (!imageDataUrl) { + setIsAnalyzing(false) + setStatusMessage(null) + setError("Could not capture frame for analysis.") + return + } + setCapturedImagePreviewUrl(imageDataUrl); // Show captured image immediately + + + setStatusMessage("Analyzing image...") + console.log("Frame captured, sending for analysis...") + + // Perform all analyses + let faceAnalysisResult = null; + try { + faceAnalysisResult = await performFaceApiAnalysis(videoRef.current); + } catch (err) { + console.error("Error in performFaceApiAnalysis for handleAnalyzeImage:", err); + } + + let mediaPipeAnalysisResult: { objectsText: string | null; gesturesText: string | null } = { objectsText: null, gesturesText: null }; + try { + mediaPipeAnalysisResult = await performMediaPipeAnalysis(videoRef.current); + let combinedObjectGestureContent = ""; + if (mediaPipeAnalysisResult.objectsText) combinedObjectGestureContent += mediaPipeAnalysisResult.objectsText; + if (mediaPipeAnalysisResult.gesturesText) { + if (combinedObjectGestureContent) combinedObjectGestureContent += "\n\n"; + combinedObjectGestureContent += mediaPipeAnalysisResult.gesturesText; + } + setObjectCardContent(combinedObjectGestureContent || "**Objects & Gestures:**\n- None detected."); + } catch (err) { + console.error("Error in performMediaPipeAnalysis for handleAnalyzeImage:", err); + setObjectCardContent("**Objects & Gestures:**\n- Analysis failed."); + } + + try { + // Using DETAILED_BASE_PROMPT for manual analysis as well + const response = await getGroqVisionAnalysis(imageDataUrl, DETAILED_BASE_PROMPT) + setAiResponse(response) + } catch (err) { + console.error("Error analyzing image with Groq (handleAnalyzeImage):", err); + const groqErrorMessage = err instanceof Error ? err.message : "An unknown error occurred during Groq analysis."; + setError(`Groq analysis failed: ${groqErrorMessage}`); + setAiResponse("Scene analysis failed."); // Show error in AI response card + } + + setShowCards(true); // Show all cards after analyses are attempted + setIsAnalyzing(false); + setStatusMessage(null); + }, [modelsReady, mediaPipeModelsReady, captureFrame, DETAILED_BASE_PROMPT, performFaceApiAnalysis, performMediaPipeAnalysis, getGroqVisionAnalysis, setAiResponse, setCapturedImagePreviewUrl, setObjectCardContent, setIdentityInfoContent, setError, setFaceApiError, setMediaPipeError, setShowCards, setStatusMessage, videoRef]); + + + // --- Flip Camera --- + const handleFlipCamera = () => { + console.log("Flipping camera...") + setIsFrontCamera((prev) => !prev) + // State update will trigger useEffect to call setupCamera + } + + // --- Microphone Toggle Logic --- + const handleMicToggle = () => { + if (!modelsReady || !mediaPipeModelsReady) { // Check both + setError("AI models are not ready yet. Please wait or refresh."); + return; + } + if (isMicListening) { + stopMicListener(); + } else { + startMicListener(); + } + } + + // --- Start Mic Listener --- + const startMicListener = async () => { + if (isMicListening || typeof navigator === 'undefined' || !navigator.mediaDevices) { + console.warn("Media devices not available or already listening.") + return + } + setError(null) + setAiResponse(null) + setStatusMessage("Listening...") + console.log("Starting microphone listener...") + + try { + // 1. Get Audio Stream + const audioStream = await navigator.mediaDevices.getUserMedia({ audio: true }) + + // 2. Create MediaRecorder (Let browser choose default MIME type) + let recorder; + try { + recorder = new MediaRecorder(audioStream); + } catch (e1) { + console.error("MediaRecorder creation failed:", e1); + alert("Could not create audio recorder. Your browser might not support it."); + audioStream.getTracks().forEach(track => track.stop()); + return; + } + mediaRecorderRef.current = recorder; + console.log("Using MediaRecorder with MIME type:", mediaRecorderRef.current.mimeType); + + // 3. Clear previous chunks + audioChunksRef.current = [] + setIsMicListening(true) // Set listening state + setIsSpeaking(false) + setIsTranscribing(false) + + // 4. Handle Data Available + mediaRecorderRef.current.ondataavailable = (event) => { + if (event.data.size > 0) { + audioChunksRef.current.push(event.data) + } + } + + // 5. Handle Recording Stop + mediaRecorderRef.current.onstop = async () => { + console.log("Mic listener stopped manually.") + const mimeType = mediaRecorderRef.current?.mimeType || 'audio/webm'; + const audioBlob = new Blob(audioChunksRef.current, { type: mimeType }); + const blobSize = audioBlob.size; + console.log(`Audio Blob Size: ${blobSize} bytes, Type: ${mimeType}`); + + // Stop tracks *immediately* after creating blob + audioStream.getTracks().forEach(track => track.stop()); + + // Reset listening state *before* processing + setIsMicListening(false) + + if (blobSize === 0) { + console.warn("Empty audio recorded.") + setStatusMessage(null) + setError("Couldn't hear anything. Please try speaking again.") + return + } + + setStatusMessage("Transcribing...") + setIsTranscribing(true) + + const recordedChunks = [...audioChunksRef.current] // Not strictly needed now, but keeping pattern + audioChunksRef.current = [] // Clear chunks immediately + + try { + const fileName = mimeType.includes('opus') ? "recording.opus" : "recording.webm"; + const audioFile = new File([audioBlob], fileName, { type: mimeType }) + const transcription = await getGroqTranscription(audioFile) + console.log("Raw Transcription Result:", transcription); + setIsTranscribing(false) + + // Improved check for invalid transcription + if (!transcription || transcription.toLowerCase().startsWith("sorry") || transcription.trim() === "" || transcription.trim().toLowerCase() === "you") { + setError("Could not understand audio or transcription was invalid. Please try again."); + console.warn("Invalid transcription received:", transcription); + setStatusMessage(null); + return; + } + + // Now that we have transcription, capture frame and analyze + await handleAnalyzeVoice(transcription) + + } catch (transcriptionError) { + console.error("Transcription error:", transcriptionError) + setError("Failed to transcribe audio.") + setStatusMessage(null) + setIsTranscribing(false) + } + } + + // 6. Start Recording + mediaRecorderRef.current.start(500) // Collect data in chunks + console.log("Microphone listener started.") + + } catch (err) { + console.error("Error accessing microphone:", err) + alert("Could not access microphone. Please check permissions.") + setStatusMessage(null) + setIsMicListening(false) + } + } + + // --- Stop Mic Listener (Manual) --- + const stopMicListener = () => { + if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") { + console.log("Manually stopping mic listener...") + mediaRecorderRef.current.stop() // This triggers the onstop handler + } else { + console.log("Mic listener was not recording.") + setIsMicListening(false) // Ensure state is reset if somehow it wasn't recording + setStatusMessage(null) + } + } + + // --- Analysis Function (Voice Input) --- + const handleAnalyzeVoice = useCallback(async (transcribedQuery: string) => { + setStatusMessage("Capturing image...") + setIsAnalyzing(true) + setAiResponse(null) + setError(null) + // Ensure all cards are hidden initially for new analysis + setCapturedImagePreviewUrl(null); + setObjectCardContent(null); + setIdentityInfoContent(null); + setShowCards(false); + setFaceApiError(null); + setMediaPipeError(null); + + const imageDataUrl = captureFrame() + + if (!imageDataUrl) { + setIsAnalyzing(false) + setStatusMessage(null) + setError("Could not capture frame for analysis.") + return + } + setCapturedImagePreviewUrl(imageDataUrl); // Show captured image immediately + + + setStatusMessage("Analyzing image with your query...") + console.log(`Frame captured, analyzing with query: "${transcribedQuery}"`) + + // Perform all analyses for voice query as well + let faceAnalysisResult = null; + try { + faceAnalysisResult = await performFaceApiAnalysis(videoRef.current); + } catch (err) { + console.error("Error in performFaceApiAnalysis for handleAnalyzeVoice:", err); + } + + let mediaPipeAnalysisResult: { objectsText: string | null; gesturesText: string | null } = { objectsText: null, gesturesText: null }; + try { + mediaPipeAnalysisResult = await performMediaPipeAnalysis(videoRef.current); + let combinedObjectGestureContent = ""; + if (mediaPipeAnalysisResult.objectsText) combinedObjectGestureContent += mediaPipeAnalysisResult.objectsText; + if (mediaPipeAnalysisResult.gesturesText) { + if (combinedObjectGestureContent) combinedObjectGestureContent += "\n\n"; + combinedObjectGestureContent += mediaPipeAnalysisResult.gesturesText; + } + setObjectCardContent(combinedObjectGestureContent || "**Objects & Gestures:**\n- None detected."); + } catch (err) { + console.error("Error in performMediaPipeAnalysis for handleAnalyzeVoice:", err); + setObjectCardContent("**Objects & Gestures:**\n- Analysis failed."); + } + + // Define the base accessibility prompt (same as in handleAnalyzeImage) + const basePrompt = "You are an AI that helps blind people by describing what you see in an image.\nSpeak clearly and simply. Write your answer in first person, like you're talking to the user.\n\nStart by saying \u201CI see...\u201D\n\nDescribe the most important things in the image. For example: people, objects, actions, places.\n\nIf there is text in the image (like signs, books, screens), read it out loud in your answer.\n\nSpeak like a helpful friend. Use short sentences.\n\nOnly say what is clearly visible. Do not guess or imagine things."; + + // Combine the base prompt with the user's query + const finalPrompt = `${basePrompt}\n\nBased on that description style, the user specifically asked: "${transcribedQuery}"` + console.log("Combined Vision Prompt:", finalPrompt); // Log the combined prompt + + try { + // Send the combined prompt to the vision analysis function + const response = await getGroqVisionAnalysis(imageDataUrl, finalPrompt) + setAiResponse(response) + setStatusMessage("Speaking response...") + // Speak the response + speakText( + response, + () => { console.log("TTS started"); setIsSpeaking(true); setStatusMessage("Speaking..."); }, + () => { + console.log("TTS ended"); + setIsSpeaking(false); + setStatusMessage(null); + // Auto-hide response and related cards after speaking + setAiResponse(null); + setCapturedImagePreviewUrl(null); + setObjectCardContent(null); + setIdentityInfoContent(null); + setShowCards(false); // Explicitly hide cards after TTS + }, + (err) => { + console.error("TTS Error:", err); + setError("Error speaking the response."); + setIsSpeaking(false); + setStatusMessage(null); + } + ); + + } catch (err) { + console.error("Error analyzing image with Groq (voice query):", err); + const groqErrorMessage = err instanceof Error ? err.message : "An unknown error occurred during Groq analysis."; + setError(`Groq analysis failed: ${groqErrorMessage}`); + setStatusMessage(null); + } finally { + setIsAnalyzing(false); + setShowCards(true); // Show cards after all analysis attempts (might be empty if errors) + } + }, [captureFrame, modelsReady, mediaPipeModelsReady, DETAILED_BASE_PROMPT, performFaceApiAnalysis, performMediaPipeAnalysis, getGroqVisionAnalysis, speakText, videoRef, setCapturedImagePreviewUrl, setObjectCardContent, setIdentityInfoContent, setAiResponse, setError, setFaceApiError, setMediaPipeError, setShowCards, setStatusMessage, setIsAnalyzing, setIsSpeaking]); + + // Main prediction loop and drawing logic + useEffect(() => { + const videoElement = videoRef.current; + const canvasElement = canvasRef.current; + let animationFrameId: number | null = null; + + const predictWebcam = async () => { + if (!videoElement || !canvasElement || !objectDetector || !gestureRecognizer || !modelsReady || !mediaPipeModelsReady || document.hidden) { + if (animationFrameId) cancelAnimationFrame(animationFrameId); + return; + } + + // Ensure this block and its contents are not causing issues. + // The face-api.js drawing part seems to be removed in latest versions, let's assume it's intentional. + // If MediaPipe drawing is needed, it would be here. + + // Request next frame + if (!document.hidden) { // Check if document is visible before requesting new frame + animationFrameId = requestAnimationFrame(predictWebcam); + } + }; + + // Start prediction loop if models are ready and video is available + if (modelsReady && mediaPipeModelsReady && videoRef.current && videoRef.current.readyState >= 2) { // readyState 2 means HAVE_CURRENT_DATA + predictWebcam(); + } + + + return () => { + if (animationFrameId) { + cancelAnimationFrame(animationFrameId); + } + // Clear canvas on component unmount or when dependencies change + if (canvasElement) { + const context = canvasElement.getContext('2d'); + if (context) { + context.clearRect(0, 0, canvasElement.width, canvasElement.height); + } + } + }; + }, [objectDetector, gestureRecognizer, modelsReady, mediaPipeModelsReady, videoRef, canvasRef, getTinyFaceDetectorOptions, areModelsLoaded, identityInfoContent, objectCardContent]); + + const handleAnalyze = useCallback(async (query?: string) => { + // ... existing code ... + }, []); + + // Check for mobile screen size + useEffect(() => { + const checkMobileSize = () => { + setIsMobile(window.innerWidth <= 768); + }; + + // Check on mount and add resize listener + checkMobileSize(); + window.addEventListener('resize', checkMobileSize); + + return () => { + window.removeEventListener('resize', checkMobileSize); + }; + }, []); + + // Swipe handlers for mobile carousel + const mobileCardHandlers = useSwipeable({ + onSwipedLeft: () => { + // Move to next card, wrapping around if at end + setActiveCardIndex((prev) => + prev < 2 ? prev + 1 : 0 + ); + }, + onSwipedRight: () => { + // Move to previous card, wrapping around if at start + setActiveCardIndex((prev) => + prev > 0 ? prev - 1 : 2 + ); + }, + preventScrollOnSwipe: true, + }); + + // Mobile card order and rendering + const mobileCards = [ + { + key: 'identity', + content: ( +

+ {/* Existing Identity Card content */} + {showCards && (capturedImagePreviewUrl || identityInfoContent) && ( + + {/* Captured Image Preview */} + {capturedImagePreviewUrl && ( +
+ Captured Frame +
+ )} + + {/* Identity Info */} + {identityInfoContent && ( +
+ {identityInfoContent.split('\n').map((trait, index) => { + const cleanTrait = trait.replace('- ', ''); + let icon = '🤔'; + if (cleanTrait.includes('Age')) icon = '🎂'; + if (cleanTrait.includes('Gender')) icon = '👤'; + if (cleanTrait.includes('Mood')) icon = '😀'; + + return ( +
+ {icon} {cleanTrait} +
+ ); + })} +
+ )} + + )} +
+ ) + }, + { + key: 'scene', + content: ( +
+ {/* Existing Scene Analysis Card content */} + {showCards && aiResponse && ( + +
+

Scene Analysis

+ {!isRecording && ( + + )} +
+
+ {aiResponse} +
+ + )} +
+ ) + }, + { + key: 'objects', + content: ( +
+ {/* Existing Objects & Gestures Card content */} + {showCards && objectCardContent && ( + +

Objects & Gestures

+ {objectCardContent && ( + <> +

Detected Objects

+
+ {objectCardContent.includes("Detected Objects:") && + objectCardContent.split("\n") + .filter(line => line.trim().startsWith("- ")) + .map((item, index) => ( + + {item.replace("- ", "")} + + )) + } +
+ + )} + + )} +
+ ) + } + ]; + + return ( +
+ {/* Mobile Top Bar - Only show on mobile */} + {isMobile && ( +
+ +
+ 📠Mumbai + 🕒 {new Date().toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} + ðŸŒ¤ï¸ 32°C +
+
+ )} + + {/* Existing camera view and other content */} +
+ {(error || faceApiError || mediaPipeError) && ( // Display general error, faceApiError or mediaPipeError +
+ +

Error Encountered

+

{error || faceApiError || mediaPipeError}

+ {/* Show Try Again only for camera errors OR model loading errors */} + {( (error && error.includes("camera")) || faceApiError || mediaPipeError ) && + + } +
+ )} + {/* Video element must be rendered even if there's an error for setupCamera to attach to */} +
+ + {/* Mobile Carousel - Only show on mobile */} + {isMobile ? ( +
+
+ {mobileCards.map((card, index) => ( +
+ {card.content} +
+ ))} +
+ + {/* Carousel Indicator Dots */} +
+ {mobileCards.map((_, index) => ( +
+ ))} +
+
+ ) : ( + // Desktop layout - existing stacked cards + <> + {/* Identity Card */} +
+ {showCards && (capturedImagePreviewUrl || identityInfoContent) && ( + + {/* Captured Image Preview */} + {capturedImagePreviewUrl && ( +
+ Captured Frame +
+ )} + + {/* Identity Info */} + {identityInfoContent && ( +
+ {identityInfoContent.split('\n').map((trait, index) => { + const cleanTrait = trait.replace('- ', ''); + let icon = '🤔'; + if (cleanTrait.includes('Age')) icon = '🎂'; + if (cleanTrait.includes('Gender')) icon = '👤'; + if (cleanTrait.includes('Mood')) icon = '😀'; + + return ( +
+ {icon} {cleanTrait} +
+ ); + })} +
+ )} + + )} +
+ + {/* Objects & Gestures Card */} +
+ {showCards && objectCardContent && ( + +

Objects & Gestures

+ {objectCardContent && ( + <> +

Detected Objects

+
+ {objectCardContent.includes("Detected Objects:") && + objectCardContent.split("\n") + .filter(line => line.trim().startsWith("- ")) + .map((item, index) => ( + + {item.replace("- ", "")} + + )) + } +
+ + )} + + )} +
+ + {/* Scene Analysis Card */} +
+ {showCards && aiResponse && ( + +
+

Scene Analysis

+ {!isRecording && ( + + )} +
+
+ {aiResponse} +
+ + )} +
+ + )} + + {/* Existing controls */} +
+ {/* Existing control buttons */} + + + {/* Camera Toggle Button */} + + + {/* Camera Button (Analyze Image) */} + + + {/* Record Button */} + + + {/* Voice Input Toggle Button */} + +
+
+ ); +} \ No newline at end of file diff --git a/app/ar-mode/page.tsx b/app/ar-mode/page.tsx index 83f2b54..fe2224c 100644 --- a/app/ar-mode/page.tsx +++ b/app/ar-mode/page.tsx @@ -3,580 +3,25 @@ import { useState, useEffect, useRef, useCallback } from "react" import { useRouter } from "next/navigation" import { Button } from "@/components/ui/button" -import { ArrowLeft, Camera, Mic, Loader2, AlertTriangle, Volume2, X, RefreshCw } from "lucide-react" +import { ArrowLeft, Camera, Mic, Loader2, AlertTriangle, Volume2, X, RefreshCw, CircleDot } from "lucide-react" import { motion, AnimatePresence } from "framer-motion" // Import the new service function import { getGroqVisionAnalysis, getGroqTranscription } from "@/lib/groq-service" import { speakText } from "@/lib/tts-service" -import ReactMarkdown from 'react-markdown' -import remarkGfm from 'remark-gfm' -import { AnimatedTooltip } from "@/components/ui/animated-tooltip" - -// Define the detailed base prompt as a constant -const DETAILED_BASE_PROMPT = `You are an AI that helps blind people by describing what you see in an image. -Speak clearly and simply. Write your answer in first person, like you're talking to the user. -Start by saying "I see…" -Describe the most important things in the image. For example: people, objects, actions, places. -If there is text in the image (like signs, books, screens), read it out loud in your answer. -Speak like a helpful friend. Use short sentences. -Only say what is clearly visible. Do not guess or imagine things. In your description, please bold keywords like person, gesture, object, location, specific cloth, some characteristic about an object, etc. YOU MAY BOLD ONLY 4 WORDS MAX using markdown. - -Example: -I see a person sitting on a bench in a park. The person is wearing a red jacket and reading a book. There is a green tree in the background. - -Be mindful not to over embellish or add any assumptions about the image. Focus solely on what is visible and described.`; - -const IDENTITY_DISCLAIMER = "Identity estimations are based on visual cues and may not be fully accurate."; - -export default function ARModePage() { - const router = useRouter() - const videoRef = useRef(null) - const canvasRef = useRef(null) - const [stream, setStream] = useState(null) - const [error, setError] = useState(null) - const [isLoading, setIsLoading] = useState(false) - const [aiResponse, setAiResponse] = useState(null) - const [isAnalyzing, setIsAnalyzing] = useState(false) - const [userQuery, setUserQuery] = useState("Describe what you see in detail.") - const [isMicListening, setIsMicListening] = useState(false) - const [isTranscribing, setIsTranscribing] = useState(false) - const [isSpeaking, setIsSpeaking] = useState(false) - const [statusMessage, setStatusMessage] = useState(null) - const [isFrontCamera, setIsFrontCamera] = useState(false) - const mediaRecorderRef = useRef(null) - const audioChunksRef = useRef([]) - const [capturedImagePreviewUrl, setCapturedImagePreviewUrl] = useState(null); - const [objectCardContent, setObjectCardContent] = useState(null); - const [identityInfoContent, setIdentityInfoContent] = useState(null); - - // Function to setup camera - const setupCamera = useCallback(async () => { - setError(null) - // Stop previous stream if it exists (important when flipping) - setStream(currentStream => { - if (currentStream) { - console.log("Stopping previous camera stream...") - currentStream.getTracks().forEach((track) => track.stop()) - } - return null; - }); - console.log(`Attempting to access ${isFrontCamera ? 'front' : 'rear'} camera...`) - try { - const mediaStream = await navigator.mediaDevices.getUserMedia({ - video: { facingMode: isFrontCamera ? "user" : "environment" }, - audio: false, - }) - setStream(mediaStream) - if (videoRef.current) { - videoRef.current.srcObject = mediaStream - } - } catch (err) { - console.error("Error accessing camera:", err) - if (err instanceof Error) { - if (err.name === "NotAllowedError") { - setError("Camera permission denied. Please enable camera access in your browser settings.") - } else if (err.name === "NotFoundError" || err.name === "DevicesNotFoundError") { - setError("No suitable camera found. Please ensure a camera is connected and enabled."); - } else { - setError(`Error accessing camera: ${err.message}`) - } - } else { - setError("An unknown error occurred while accessing the camera.") - } - } - }, [isFrontCamera]) - - // Setup camera on mount and when isFrontCamera changes - useEffect(() => { - setupCamera() - - // Cleanup stream on unmount - return () => { - setStream(currentStream => { - if (currentStream) { - console.log("Cleaning up camera stream...") - currentStream.getTracks().forEach((track) => track.stop()) - } - return null; // Ensure stream state is reset - }); - } - }, [setupCamera, isFrontCamera]) - - // Function to capture frame - const captureFrame = useCallback((): string | null => { - if (!videoRef.current || !canvasRef.current) { - console.error("Video or canvas ref not available") - return null - } - const video = videoRef.current - const canvas = canvasRef.current - canvas.width = video.videoWidth - canvas.height = video.videoHeight - const context = canvas.getContext("2d") - if (!context) { - console.error("Could not get canvas context") - return null - } - context.drawImage(video, 0, 0, canvas.width, canvas.height) - // Convert to base64 JPEG - try { - return canvas.toDataURL("image/jpeg", 0.9) // Adjust quality as needed - } catch (e) { - console.error("Error converting canvas to data URL:", e) - setError("Failed to capture frame.") - return null - } - }, []) - - // Function to handle analysis - const handleAnalyzeImage = useCallback(async () => { - setStatusMessage("Capturing image...") - setIsAnalyzing(true) - setAiResponse(null) - setCapturedImagePreviewUrl(null); - setObjectCardContent(null); - setIdentityInfoContent(null); - setError(null) - const imageDataUrl = captureFrame() - - if (!imageDataUrl) { - setIsAnalyzing(false) - setStatusMessage(null) - setError("Could not capture frame for analysis.") - return - } - setCapturedImagePreviewUrl(imageDataUrl); - - setStatusMessage("Analyzing image...") - console.log("Frame captured, sending for analysis...") - - try { - const response = await getGroqVisionAnalysis(imageDataUrl, DETAILED_BASE_PROMPT) - setAiResponse(response) - - // Populate identity info - setIdentityInfoContent("**Detected Person:**\n- Age: ~25-30\n- Gender: Male\n- Mood: Neutral"); - // Populate other objects info - setObjectCardContent("**Other Objects:**\n- **Desk Lamp** (On)\n- **Smartphone** (Screen off)"); - - setStatusMessage(null) - } catch (err) { - console.error("Error analyzing image:", err) - const errorMessage = err instanceof Error ? err.message : "An unknown error occurred during analysis." - setError(`Analysis failed: ${errorMessage}`) - setStatusMessage(null) - } finally { - setIsAnalyzing(false) - } - }, [captureFrame]) - - // --- Flip Camera --- - const handleFlipCamera = () => { - console.log("Flipping camera...") - setIsFrontCamera((prev) => !prev) - // State update will trigger useEffect to call setupCamera - } - - // --- Microphone Toggle Logic --- - const handleMicToggle = () => { - if (isMicListening) { - stopMicListener(); - } else { - startMicListener(); - } - } - - // --- Start Mic Listener --- - const startMicListener = async () => { - if (isMicListening || typeof navigator === 'undefined' || !navigator.mediaDevices) { - console.warn("Media devices not available or already listening.") - return - } - setError(null) - setAiResponse(null) - setStatusMessage("Listening...") - console.log("Starting microphone listener...") - - try { - // 1. Get Audio Stream - const audioStream = await navigator.mediaDevices.getUserMedia({ audio: true }) - - // 2. Create MediaRecorder (Let browser choose default MIME type) - let recorder; - try { - recorder = new MediaRecorder(audioStream); - } catch (e1) { - console.error("MediaRecorder creation failed:", e1); - alert("Could not create audio recorder. Your browser might not support it."); - audioStream.getTracks().forEach(track => track.stop()); - return; - } - mediaRecorderRef.current = recorder; - console.log("Using MediaRecorder with MIME type:", mediaRecorderRef.current.mimeType); - - // 3. Clear previous chunks - audioChunksRef.current = [] - setIsMicListening(true) // Set listening state - setIsSpeaking(false) - setIsTranscribing(false) - - // 4. Handle Data Available - mediaRecorderRef.current.ondataavailable = (event) => { - if (event.data.size > 0) { - audioChunksRef.current.push(event.data) - } - } - - // 5. Handle Recording Stop - mediaRecorderRef.current.onstop = async () => { - console.log("Mic listener stopped manually.") - const mimeType = mediaRecorderRef.current?.mimeType || 'audio/webm'; - const audioBlob = new Blob(audioChunksRef.current, { type: mimeType }); - const blobSize = audioBlob.size; - console.log(`Audio Blob Size: ${blobSize} bytes, Type: ${mimeType}`); - - // Stop tracks *immediately* after creating blob - audioStream.getTracks().forEach(track => track.stop()); - - // Reset listening state *before* processing - setIsMicListening(false) - - if (blobSize === 0) { - console.warn("Empty audio recorded.") - setStatusMessage(null) - setError("Couldn't hear anything. Please try speaking again.") - return - } - - setStatusMessage("Transcribing...") - setIsTranscribing(true) - - const recordedChunks = [...audioChunksRef.current] // Not strictly needed now, but keeping pattern - audioChunksRef.current = [] // Clear chunks immediately - - try { - const fileName = mimeType.includes('opus') ? "recording.opus" : "recording.webm"; - const audioFile = new File([audioBlob], fileName, { type: mimeType }) - const transcription = await getGroqTranscription(audioFile) - console.log("Raw Transcription Result:", transcription); - setIsTranscribing(false) - - // Improved check for invalid transcription - if (!transcription || transcription.toLowerCase().startsWith("sorry") || transcription.trim() === "" || transcription.trim().toLowerCase() === "you") { - setError("Could not understand audio or transcription was invalid. Please try again."); - console.warn("Invalid transcription received:", transcription); - setStatusMessage(null); - return; - } - - // Now that we have transcription, capture frame and analyze - await handleAnalyzeVoice(transcription) - - } catch (transcriptionError) { - console.error("Transcription error:", transcriptionError) - setError("Failed to transcribe audio.") - setStatusMessage(null) - setIsTranscribing(false) - } - } - - // 6. Start Recording - mediaRecorderRef.current.start(500) // Collect data in chunks - console.log("Microphone listener started.") - - } catch (err) { - console.error("Error accessing microphone:", err) - alert("Could not access microphone. Please check permissions.") - setStatusMessage(null) - setIsMicListening(false) - } - } - - // --- Stop Mic Listener (Manual) --- - const stopMicListener = () => { - if (mediaRecorderRef.current && mediaRecorderRef.current.state === "recording") { - console.log("Manually stopping mic listener...") - mediaRecorderRef.current.stop() // This triggers the onstop handler - } else { - console.log("Mic listener was not recording.") - setIsMicListening(false) // Ensure state is reset if somehow it wasn't recording - setStatusMessage(null) - } - } - - // --- Analysis Function (Voice Input) --- - const handleAnalyzeVoice = useCallback(async (transcribedQuery: string) => { - setStatusMessage("Capturing image...") - setIsAnalyzing(true) - setAiResponse(null) - setCapturedImagePreviewUrl(null); - setObjectCardContent(null); - setIdentityInfoContent(null); - setError(null) - const imageDataUrl = captureFrame() - - if (!imageDataUrl) { - setIsAnalyzing(false) - setStatusMessage(null) - setError("Could not capture frame for analysis.") - return - } - setCapturedImagePreviewUrl(imageDataUrl); - - setStatusMessage("Analyzing image with your query...") - console.log(`Frame captured, analyzing with query: "${transcribedQuery}"`) - - // Use the new detailed base prompt here as well - const finalPrompt = `${DETAILED_BASE_PROMPT}\n\nBased on that description style, the user specifically asked: "${transcribedQuery}"` - console.log("Combined Vision Prompt:", finalPrompt); // Log the combined prompt - - try { - const response = await getGroqVisionAnalysis(imageDataUrl, finalPrompt) - setAiResponse(response) - - // Populate identity info - setIdentityInfoContent("**Detected Person:**\n- Age: ~30-35\n- Gender: Female\n- Mood: Focused"); - // Populate other objects info - setObjectCardContent("**Other Objects:**\n- **Laptop** (Open)\n- **Coffee Mug** (Full)"); - - setStatusMessage("Speaking response...") - // Speak the response - speakText( - response, - () => { console.log("TTS started"); setIsSpeaking(true); setStatusMessage("Speaking..."); }, - () => { console.log("TTS ended"); setIsSpeaking(false); setStatusMessage(null); setAiResponse(null); /* Auto-hide response */ }, - (err) => { - console.error("TTS Error:", err); - setError("Error speaking the response."); - setIsSpeaking(false); - setStatusMessage(null); - } - ); - - } catch (err) { - console.error("Error analyzing image with voice query:", err) - const errorMessage = err instanceof Error ? err.message : "An unknown error occurred during analysis." - setError(`Analysis failed: ${errorMessage}`) - setStatusMessage(null) - } finally { - setIsAnalyzing(false) - // Don't set isSpeaking false here, TTS callback handles it - } - }, [captureFrame]) // Removed userQuery dependency - - return ( -
- {/* Header */} -
- -

AR Mode

- {/* Placeholder for potential settings/flash toggle */} -
-
- - {/* Camera View */} -
{/* Ensure container has a background */} - {error ? ( -
- -

Error

{/* Simplified title */} -

{error}

- {/* Conditionally show Try Again button only for camera errors */} - {error.includes("camera") && - - } -
- ) : ( -
- - {/* Top-Left Captured Image Preview Card - Now includes identity info */} - - {capturedImagePreviewUrl && ( - - Captured scene - {identityInfoContent && ( -
- {identityInfoContent} -
- -
-
- )} - - )} - - - {/* Object Card (Bottom-Left) - Now only for 'Other Objects' */} - - {objectCardContent && ( - - {/* Optional: Title for this card if it makes sense e.g. Other Details */} - {/*

Other Details

*/} -
- {objectCardContent} -
- - )} - - - {/* AI Response Overlay (Bottom Card - Scene Analysis) */} - - {aiResponse && ( - -
-
-

Scene Analysis

-
- -
- -
- {aiResponse} -
- - )} - - - {/* Controls */} -
- {/* Camera Toggle Button */} - - - {/* Camera Button */} - - - {/* Voice Input Toggle Button */} - - -
+import dynamic from 'next/dynamic' + +const ARModeContent = dynamic(() => import('@/app/ar-mode/ar-mode-content'), { + ssr: false, + loading: () => ( +
+ +

Loading AR Experience...

- ) + ), +}) + +export default function ARModePageLoader() { + // The dynamic import with ssr:false handles client-side rendering. + // The loading component from dynamic import will be shown initially. + return ; } diff --git a/app/chat/[id]/page.tsx b/app/chat/[id]/page.tsx index b863dbb..1cbe04c 100644 --- a/app/chat/[id]/page.tsx +++ b/app/chat/[id]/page.tsx @@ -1151,32 +1151,11 @@ export default function ChatPage() { )} {/* Input area & Image Preview */} -
- {/* Image Preview Section */} - {imageBase64 && ( -
-
- Selected preview - -
-
- )} - {/* End Image Preview Section */} - +
{/* Main input container - ChatGPT style rounded pill */} -
+
{/* Left side buttons */}
@@ -1230,7 +1209,7 @@ export default function ChatPage() {
{/* Input field */} -
+
void; +} + +const Switch: React.FC = ({ isToggled, onToggle }) => { + return ( + +
+ +