From ec70fe8d4f6dc6bfe70fdb9a97602279b2a9bdb9 Mon Sep 17 00:00:00 2001 From: NabilThange Date: Wed, 21 May 2025 12:40:02 +0530 Subject: [PATCH 1/9] try2 all cards and ar-mode exp --- app/ar-mode/page.tsx | 1123 +++++-- app/chat/[id]/page.tsx | 6 +- app/layout.tsx | 8 +- components/cta.tsx | 25 +- components/hero.tsx | 19 +- components/ui/animated-tooltip.tsx | 104 + components/ui/flip-words.tsx | 98 + components/ui/following-pointer.tsx | 136 + components/ui/link-preview.tsx | 152 + faceapi-docs.md | 855 +++++ lib/face-service.ts | 85 + package-lock.json | 2962 ++++++++++++++--- package.json | 9 +- pnpm-lock.yaml | 1257 ++++++- public/images/git.png | Bin 0 -> 307763 bytes public/images/info-icon.png | Bin 0 -> 1107 bytes public/images/nb-lnk.png | Bin 0 -> 249382 bytes .../efficientdet_lite0.tflite | Bin 0 -> 7254339 bytes .../mediapipe-models/gesture_recognizer.task | Bin 0 -> 8373440 bytes .../age_gender_model/age_gender_model-shard1 | Bin 0 -> 429708 bytes .../age_gender_model-weights_manifest.json | 1 + .../face_expression_model-shard1 | Bin 0 -> 329468 bytes ...ace_expression_model-weights_manifest.json | 1 + .../face_landmark_68_model-shard1 | Bin 0 -> 356840 bytes ...ce_landmark_68_model-weights_manifest.json | 1 + .../age_gender_model-weights_manifest.json | 62 + public/models/faceapi/age_gender_model.bin | Bin 0 -> 429708 bytes .../tiny_face_detector_model-shard1 | Bin 0 -> 193321 bytes ..._face_detector_model-weights_manifest.json | 1 + 29 files changed, 6128 insertions(+), 777 deletions(-) create mode 100644 components/ui/animated-tooltip.tsx create mode 100644 components/ui/flip-words.tsx create mode 100644 components/ui/following-pointer.tsx create mode 100644 components/ui/link-preview.tsx create mode 100644 faceapi-docs.md create mode 100644 lib/face-service.ts create mode 100644 public/images/git.png create mode 100644 public/images/info-icon.png create mode 100644 public/images/nb-lnk.png create mode 100644 public/mediapipe-models/efficientdet_lite0.tflite create mode 100644 public/mediapipe-models/gesture_recognizer.task create mode 100644 public/models/age_gender_model/age_gender_model-shard1 create mode 100644 public/models/age_gender_model/age_gender_model-weights_manifest.json create mode 100644 public/models/face_expression/face_expression_model-shard1 create mode 100644 public/models/face_expression/face_expression_model-weights_manifest.json create mode 100644 public/models/face_landmark_68/face_landmark_68_model-shard1 create mode 100644 public/models/face_landmark_68/face_landmark_68_model-weights_manifest.json create mode 100644 public/models/faceapi/age_gender_model-weights_manifest.json create mode 100644 public/models/faceapi/age_gender_model.bin create mode 100644 public/models/tiny_face_detector/tiny_face_detector_model-shard1 create mode 100644 public/models/tiny_face_detector/tiny_face_detector_model-weights_manifest.json diff --git a/app/ar-mode/page.tsx b/app/ar-mode/page.tsx index bed40f1..a9df7b7 100644 --- a/app/ar-mode/page.tsx +++ b/app/ar-mode/page.tsx @@ -3,11 +3,39 @@ import { useState, useEffect, useRef, useCallback } from "react" import { useRouter } from "next/navigation" import { Button } from "@/components/ui/button" -import { ArrowLeft, Camera, Mic, Loader2, AlertTriangle, Volume2, X, RefreshCw } from "lucide-react" +import { ArrowLeft, Camera, Mic, Loader2, AlertTriangle, Volume2, X, RefreshCw, CircleDot } from "lucide-react" import { motion, AnimatePresence } from "framer-motion" // Import the new service function import { getGroqVisionAnalysis, getGroqTranscription } from "@/lib/groq-service" import { speakText } from "@/lib/tts-service" +import ReactMarkdown from 'react-markdown' +import remarkGfm from 'remark-gfm' +import { AnimatedTooltip } from "@/components/ui/animated-tooltip" +import * as faceapi from 'face-api.js'; +import { loadModels, getTinyFaceDetectorOptions, areModelsLoaded } from "../../lib/face-service"; +import { + ObjectDetector, + GestureRecognizer, + FilesetResolver, + Detection, + GestureRecognizerResult +} from "@mediapipe/tasks-vision"; + +// Define the detailed base prompt as a constant +const DETAILED_BASE_PROMPT = `You are an AI that helps blind people by describing what you see in an image. +Speak clearly and simply. Write your answer in first person, like you're talking to the user. +Start by saying "I see…" +Describe the most important things in the image. For example: people, objects, actions, places. +If there is text in the image (like signs, books, screens), read it out loud in your answer. +Speak like a helpful friend. Use short sentences. +Only say what is clearly visible. Do not guess or imagine things. In your description, please bold keywords like person, gesture, object, location, specific cloth, some characteristic about an object, etc. YOU MAY BOLD ONLY 4 WORDS MAX using markdown. + +Example: +I see a person sitting on a bench in a park. The person is wearing a red jacket and reading a book. There is a green tree in the background. + +Be mindful not to over embellish or add any assumptions about the image. Focus solely on what is visible and described.`; + +const IDENTITY_DISCLAIMER = "Identity estimations are based on visual cues and may not be fully accurate."; export default function ARModePage() { const router = useRouter() @@ -26,6 +54,145 @@ export default function ARModePage() { const [isFrontCamera, setIsFrontCamera] = useState(false) const mediaRecorderRef = useRef(null) const audioChunksRef = useRef([]) + const [capturedImagePreviewUrl, setCapturedImagePreviewUrl] = useState(null); + const [objectCardContent, setObjectCardContent] = useState(null); + const [identityInfoContent, setIdentityInfoContent] = useState(null); + const [modelsReady, setModelsReady] = useState(false); + const [faceApiError, setFaceApiError] = useState(null); + const [isClient, setIsClient] = useState(false); + + // MediaPipe specific states + const [objectDetector, setObjectDetector] = useState(null); + const [gestureRecognizer, setGestureRecognizer] = useState(null); + const [mediaPipeModelsReady, setMediaPipeModelsReady] = useState(false); + const [mediaPipeError, setMediaPipeError] = useState(null); + const lastVideoTimeRef = useRef(-1); + + // Smart Record feature - new states + const [isRecording, setIsRecording] = useState(false); + const [recordingTime, setRecordingTime] = useState(0); + const [recordStartTime, setRecordStartTime] = useState(null); + const recordingIntervalRef = useRef(null); + const captureIntervalRef = useRef(null); + + // Card visibility states - explicitly track whether cards should be shown + const [showCards, setShowCards] = useState(false); + const [lastAnalysisTime, setLastAnalysisTime] = useState(null); + + // Effect to set client-side flag and prevent hydration mismatches + useEffect(() => { + setIsClient(true); + }, []); + + // Modify existing model loading effect + useEffect(() => { + const initFaceApi = async () => { + // Only attempt to load on client + if (!isClient) return; + + try { + setStatusMessage("Loading AI models..."); + setError(null); + setFaceApiError(null); + + // Check if models are already loaded to prevent redundant loading + if (!areModelsLoaded()) { + await loadModels(); + } + + setModelsReady(true); + setStatusMessage(null); + console.log("Face-API models loaded and ready."); + } catch (err) { + console.error("Failed to load face-API models:", err); + const errorMessage = err instanceof Error + ? err.message + : "Unknown error loading face models"; + + setFaceApiError(errorMessage); + setError(`Failed to initialize AI features: ${errorMessage}. Please try refreshing the page.`); + setStatusMessage(null); + } + }; + + initFaceApi(); + }, [isClient]); // Depend on isClient to ensure client-side only + + // Effect to load MediaPipe models + useEffect(() => { + const initMediaPipe = async () => { + if (!isClient) return; // Only run on client + + // Ensure Face-API models are loaded or had a chance to load first + // This helps in sequencing the loading messages + if (!modelsReady && !faceApiError) { + // If Face-API is still loading, wait for it. + // This effect will re-run when modelsReady changes. + return; + } + + setStatusMessage("Loading MediaPipe models..."); + setMediaPipeError(null); + + try { + const vision = await FilesetResolver.forVisionTasks( + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" + ); + + // Initialize Object Detector + const loadedObjectDetector = await ObjectDetector.createFromOptions(vision, { + baseOptions: { + modelAssetPath: `/mediapipe-models/efficientdet_lite0.tflite`, // Corrected path + delegate: "GPU" + }, + scoreThreshold: 0.5, + runningMode: "VIDEO", // Corrected: Use "VIDEO" for detectForVideo + }); + setObjectDetector(loadedObjectDetector); + console.log("MediaPipe Object Detector loaded."); + + // Initialize Gesture Recognizer + const loadedGestureRecognizer = await GestureRecognizer.createFromOptions(vision, { + baseOptions: { + modelAssetPath: `/mediapipe-models/gesture_recognizer.task`, // Corrected path + delegate: "GPU" + }, + runningMode: "VIDEO", // Corrected: Use "VIDEO" for recognizeForVideo + numHands: 2 + }); + setGestureRecognizer(loadedGestureRecognizer); + console.log("MediaPipe Gesture Recognizer loaded."); + + setMediaPipeModelsReady(true); + setStatusMessage(null); // All models loaded + console.log("All AI models (Face-API & MediaPipe) are ready."); + + } catch (err) { + console.error("Failed to load MediaPipe models:", err); + const errorMessage = err instanceof Error ? err.message : "Unknown error loading MediaPipe models"; + setMediaPipeError(errorMessage); + setError(prevError => prevError ? `${prevError} & MediaPipe failed: ${errorMessage}` : `MediaPipe models failed: ${errorMessage}. Try refreshing.`); + setStatusMessage(null); + } + }; + + initMediaPipe(); + + // Cleanup MediaPipe tasks on unmount + return () => { + console.log("Closing MediaPipe tasks..."); + if (objectDetector) { + objectDetector.close(); + setObjectDetector(null); + } + if (gestureRecognizer) { + gestureRecognizer.close(); + setGestureRecognizer(null); + } + setMediaPipeModelsReady(false); + }; + // Add modelsReady and faceApiError to dependencies to run after face-api init. + }, [isClient, modelsReady, faceApiError]); // Function to setup camera const setupCamera = useCallback(async () => { @@ -66,8 +233,9 @@ export default function ARModePage() { // Setup camera on mount and when isFrontCamera changes useEffect(() => { - setupCamera() - + if (isClient) { // Ensure this runs only on client-side + setupCamera(); + } // Cleanup stream on unmount return () => { setStream(currentStream => { @@ -78,7 +246,7 @@ export default function ARModePage() { return null; // Ensure stream state is reset }); } - }, [setupCamera, isFrontCamera]) + }, [setupCamera, isClient]); // Function to capture frame const captureFrame = useCallback((): string | null => { @@ -106,41 +274,392 @@ export default function ARModePage() { } }, []) - // Function to handle analysis + // Helper function for Face-API.js analysis - now for one-time analysis on a given media element + const performFaceApiAnalysis = useCallback(async (mediaElement: HTMLVideoElement | HTMLImageElement | HTMLCanvasElement | null) => { + if (!modelsReady || !mediaElement) { + console.warn("Face-API models not ready or media element not available for analysis."); + setIdentityInfoContent(prev => prev === null ? `**Identity:**\n- Models not ready` : prev); + return null; + } + + // Check if it's a video element and if it's ready + if (mediaElement instanceof HTMLVideoElement && (mediaElement.readyState < mediaElement.HAVE_METADATA || mediaElement.paused || mediaElement.ended)) { + console.warn("Video element not ready, paused, or ended for Face-API analysis."); + setIdentityInfoContent(prev => prev === null ? `**Identity:**\n- Video not ready` : prev); + return null; + } + + setFaceApiError(null); + + try { + console.log("Performing face-api.js detection..."); + const detections = await faceapi + .detectAllFaces(mediaElement, getTinyFaceDetectorOptions()) + .withFaceLandmarks() + .withFaceExpressions() + .withAgeAndGender(); + + if (detections && detections.length > 0) { + const firstDetection = detections[0]; + const { age, gender, expressions } = firstDetection; + + let dominantExpression = "neutral"; + let maxProbability = 0; + + if (expressions && typeof expressions === 'object') { + for (const expressionKey in expressions) { + if (Object.prototype.hasOwnProperty.call(expressions, expressionKey)) { + const probability = (expressions as any)[expressionKey]; + if (typeof probability === 'number' && probability > maxProbability) { + maxProbability = probability; + dominantExpression = expressionKey; + } + } + } + } + + const identityText = `**Detected Person:**\n- Age: ~${Math.round(age)} years\n- Gender: ${gender.charAt(0).toUpperCase() + gender.slice(1)}\n- Mood: ${dominantExpression.charAt(0).toUpperCase() + dominantExpression.slice(1)}`; + setIdentityInfoContent(identityText); + console.log("One-time Face-API Analysis Result:", { age: Math.round(age), gender, mood: dominantExpression }); + return { age: Math.round(age), gender, mood: dominantExpression }; + } else { + // Improved message for no face detection + const blurryMessage = "**No Clear Face Detected:**\n- Image might be blurry\n- Face not in frame\n- Poor lighting conditions\n\n*Cannot conclude identity details*"; + setIdentityInfoContent(blurryMessage); + console.log("No human faces detected by Face-API in the captured image."); + return null; + } + } catch (err) { + console.error("Error during one-time face-api.js detection:", err); + const errorMessage = err instanceof Error ? err.message : "Face detection failed on captured image"; + + // Specific error handling message + const errorText = `**Detection Error:**\n- ${errorMessage}\n- Try adjusting camera\n- Ensure good lighting`; + + setFaceApiError(errorMessage); + setIdentityInfoContent(errorText); + return null; + } + }, [modelsReady, getTinyFaceDetectorOptions, setIdentityInfoContent, setFaceApiError]); + + // --- New function for MediaPipe Analysis --- + const performMediaPipeAnalysis = useCallback(async (videoElement: HTMLVideoElement | null): Promise<{ objectsText: string | null; gesturesText: string | null }> => { + if (!mediaPipeModelsReady || !objectDetector || !gestureRecognizer || !videoElement) { + console.warn("MediaPipe models not ready or video element not available for analysis."); + return { objectsText: `**Objects/Gestures:**\n- MediaPipe not ready`, gesturesText: null }; + } + + if (videoElement.readyState < videoElement.HAVE_METADATA || videoElement.paused || videoElement.ended) { + console.warn("Video element not ready, paused, or ended for MediaPipe analysis."); + return { objectsText: `**Objects/Gestures:**\n- Video not ready`, gesturesText: null }; + } + + // Prevent running if the video frame hasn't changed + // if (videoElement.currentTime === lastVideoTimeRef.current) { + // console.log("Skipping MediaPipe analysis, video frame unchanged."); + // // Return previously set content or null if you want to force update + // return { objectsText: null, gesturesText: null }; + // } // Temporarily disabling this check as it might interfere with interval logic + lastVideoTimeRef.current = videoElement.currentTime; + + + let objectsText: string | null = null; + let gesturesText: string | null = null; + + try { + // Object Detection + const objectDetections = objectDetector.detectForVideo(videoElement, performance.now()); + if (objectDetections && objectDetections.detections.length > 0) { + const detectedObjectNames = objectDetections.detections + .map(det => det.categories[0]?.categoryName || "Unknown Object") + .filter((name, index, self) => self.indexOf(name) === index); // Unique names + + if (detectedObjectNames.length > 0) { + objectsText = `**Detected Objects:**\n- ${detectedObjectNames.join('\n- ')}`; + } else { + objectsText = "**Detected Objects:**\n- None clearly identified"; + } + console.log("MediaPipe Object Detection Result:", detectedObjectNames); + } else { + objectsText = "**Detected Objects:**\n- None"; + console.log("No objects detected by MediaPipe."); + } + + // Gesture Recognition + const gestureRecognitionResult: GestureRecognizerResult = gestureRecognizer.recognizeForVideo(videoElement, performance.now()); + if (gestureRecognitionResult.gestures.length > 0) { + const gestureMap: { [key: string]: string } = { + "None": "None", + "Closed_Fist": "Closed Fist ✊", + "Open_Palm": "Open Palm ðŸ–ï¸", + "Pointing_Up": "Pointing Up â˜ï¸", + "Thumb_Down": "Thumbs Down 👎", + "Thumb_Up": "Thumbs Up ðŸ‘", + "Victory": "Victory ✌ï¸", + "ILoveYou": "I Love You 🤟", + "Unknown": "Unknown Gesture â”" + }; + + const recognizedGestures = gestureRecognitionResult.gestures + .map(gestureData => { + const categoryName = gestureData[0]?.categoryName || "Unknown"; + return gestureMap[categoryName] || `${categoryName} (Unknown)`; // Fallback for unmapped gestures + }) + .filter((name, index, self) => self.indexOf(name) === index && name !== "None"); // Unique and not "None" + + if (recognizedGestures.length > 0) { + gesturesText = `**Detected Gestures:**\n- ${recognizedGestures.join('\n- ')}`; + } else { + gesturesText = "**Detected Gestures:**\n- None clearly identified"; + } + console.log("MediaPipe Gesture Recognition Result:", recognizedGestures); + } else { + gesturesText = "**Detected Gestures:**\n- None"; + console.log("No gestures detected by MediaPipe."); + } + + } catch (err) { + console.error("Error during MediaPipe analysis:", err); + const mediaPipeErrorMessage = err instanceof Error ? err.message : "MediaPipe analysis failed"; + setMediaPipeError(mediaPipeErrorMessage); // Set specific MediaPipe error + // Optionally, update objectsText/gesturesText to show error + objectsText = `**Object Detection Error:**\n- ${mediaPipeErrorMessage}`; + gesturesText = `**Gesture Recognition Error:**\n- ${mediaPipeErrorMessage}`; + } + + return { objectsText, gesturesText }; + }, [mediaPipeModelsReady, objectDetector, gestureRecognizer, setMediaPipeError, lastVideoTimeRef]); + + // Format recording time as MM:SS + const formatRecordingTime = (seconds: number): string => { + const mins = Math.floor(seconds / 60).toString().padStart(2, '0'); + const secs = (seconds % 60).toString().padStart(2, '0'); + return `${mins}:${secs}`; + }; + + // Capture and analyze a single frame + const captureAndAnalyzeFrame = useCallback(async () => { + console.log("[captureAndAnalyzeFrame] Starting frame analysis (called from interval or initial trigger)."); + + const imageDataUrl = captureFrame(); + if (!imageDataUrl) { + console.error("[captureAndAnalyzeFrame] Could not capture frame."); + return; + } + + setCapturedImagePreviewUrl(imageDataUrl); + console.log("[captureAndAnalyzeFrame] Captured image preview URL set."); + + try { + await performFaceApiAnalysis(videoRef.current); + console.log("[captureAndAnalyzeFrame] Face analysis complete."); + } catch (err) { + console.error("[captureAndAnalyzeFrame] Face analysis error:", err); + } + + try { + const { objectsText, gesturesText } = await performMediaPipeAnalysis(videoRef.current); + let combinedObjectGestureContent = ""; + if (objectsText) combinedObjectGestureContent += objectsText; + if (gesturesText) { + if (combinedObjectGestureContent) combinedObjectGestureContent += "\n\n"; + combinedObjectGestureContent += gesturesText; + } + if (!combinedObjectGestureContent) { + combinedObjectGestureContent = "**Objects & Gestures:**\n- None detected or analysis issue."; + } + setObjectCardContent(combinedObjectGestureContent); + console.log("[captureAndAnalyzeFrame] MediaPipe analysis complete. Content:", combinedObjectGestureContent); + } catch (err) { + console.error("[captureAndAnalyzeFrame] MediaPipe analysis error:", err); + } + + try { + const response = await getGroqVisionAnalysis(imageDataUrl, DETAILED_BASE_PROMPT); + setAiResponse(response); + console.log("[captureAndAnalyzeFrame] Groq vision analysis complete. Response:", response); + } catch (err) { + console.error("[captureAndAnalyzeFrame] Groq vision analysis error:", err); + } + + setShowCards(true); + setLastAnalysisTime(Date.now()); + console.log("[captureAndAnalyzeFrame] setShowCards(true) called. lastAnalysisTime set."); + }, [ + captureFrame, + DETAILED_BASE_PROMPT, + performFaceApiAnalysis, + performMediaPipeAnalysis, + getGroqVisionAnalysis, + setCapturedImagePreviewUrl, + setIdentityInfoContent, + setObjectCardContent, + setAiResponse, + setShowCards, + setLastAnalysisTime, + videoRef // videoRef.current is used by analysis helpers + ]); + + // Effect to handle recording logic (timer and periodic capture) + useEffect(() => { + let localRecordingTimerId: NodeJS.Timeout | null = null; + let localFrameAnalysisIntervalId: NodeJS.Timeout | null = null; + + if (isRecording) { + console.log("[Recording Effect] Recording started. Performing initial capture."); + setStatusMessage("Smart recording initializing..."); + + captureAndAnalyzeFrame().then(() => { + setStatusMessage("Smart recording..."); + }); + + localRecordingTimerId = setInterval(() => { + setRecordingTime(prev => prev + 1); + }, 1000); + recordingIntervalRef.current = localRecordingTimerId; + + localFrameAnalysisIntervalId = setInterval(() => { + console.log("[Recording Effect] Interval: Capturing and analyzing frame."); + captureAndAnalyzeFrame(); + }, 4000); + captureIntervalRef.current = localFrameAnalysisIntervalId; + + setRecordStartTime(Date.now()); + + } else { + console.log("[Recording Effect] Recording stopped."); + if (recordingIntervalRef.current) { + clearInterval(recordingIntervalRef.current); + recordingIntervalRef.current = null; + } + if (captureIntervalRef.current) { + clearInterval(captureIntervalRef.current); + captureIntervalRef.current = null; + } + setRecordingTime(0); + setRecordStartTime(null); + if (statusMessage === "Smart recording..." || statusMessage === "Smart recording initializing...") { + setStatusMessage(null); + } + } + + return () => { + console.log("[Recording Effect] Cleanup: Clearing intervals from effect."); + if (localRecordingTimerId) clearInterval(localRecordingTimerId); + if (localFrameAnalysisIntervalId) clearInterval(localFrameAnalysisIntervalId); + + if (recordingIntervalRef.current === localRecordingTimerId) recordingIntervalRef.current = null; + if (captureIntervalRef.current === localFrameAnalysisIntervalId) captureIntervalRef.current = null; + }; + }, [isRecording, captureAndAnalyzeFrame, setStatusMessage, setRecordingTime, setRecordStartTime]); + + // Handle Record Toggle + const handleRecordToggle = useCallback(() => { + if (!modelsReady || !mediaPipeModelsReady) { + setError("AI models are not ready yet. Please wait or refresh."); + return; + } + + setIsRecording(prevIsRecording => { + const newIsRecording = !prevIsRecording; + if (newIsRecording) { + console.log("[handleRecordToggle] Starting recording process..."); + setCapturedImagePreviewUrl(null); + setObjectCardContent(null); + setIdentityInfoContent(null); + setAiResponse(null); + setShowCards(false); + setLastAnalysisTime(null); + } else { + console.log("[handleRecordToggle] Stopping recording process..."); + } + return newIsRecording; + }); + }, [modelsReady, mediaPipeModelsReady, setIsRecording, setCapturedImagePreviewUrl, setObjectCardContent, setIdentityInfoContent, setAiResponse, setShowCards, setLastAnalysisTime, setError]); + + // Handle Analyze Image (one-time capture) const handleAnalyzeImage = useCallback(async () => { - setStatusMessage("Capturing image...") - setIsAnalyzing(true) - setAiResponse(null) - setError(null) - const imageDataUrl = captureFrame() + if (!modelsReady || !mediaPipeModelsReady) { // Check both Face-API and MediaPipe models + setError("AI models are not ready yet. Please wait or refresh."); + return; + } + setStatusMessage("Processing..."); + setIsAnalyzing(true); + setAiResponse(null); + setCapturedImagePreviewUrl(null); + setObjectCardContent(null); + setIdentityInfoContent(null); // Clear previous identity info for new analysis + setError(null); + setFaceApiError(null); + setMediaPipeError(null); // Clear MediaPipe error + setShowCards(false); // Reset card visibility until analysis completes + // Capture frame + const imageDataUrl = captureFrame(); if (!imageDataUrl) { - setIsAnalyzing(false) - setStatusMessage(null) - setError("Could not capture frame for analysis.") - return + setIsAnalyzing(false); + setStatusMessage(null); + setError("Could not capture frame for analysis."); + return; } + + // Always ensure image preview is visible + setCapturedImagePreviewUrl(imageDataUrl); - setStatusMessage("Analyzing image...") - console.log("Frame captured, sending for analysis...") + // Run face analysis + setStatusMessage("Analyzing faces..."); + try { + await performFaceApiAnalysis(videoRef.current); + } catch (err) { + console.error("Face analysis error:", err); + const errorMessage = err instanceof Error ? err.message : "Unknown error"; + setFaceApiError(`Face analysis failed: ${errorMessage}`); + } + + // Run MediaPipe analysis + setStatusMessage("Analyzing objects & gestures..."); + try { + const { objectsText, gesturesText } = await performMediaPipeAnalysis(videoRef.current); + + let combinedObjectGestureContent = ""; + if (objectsText) combinedObjectGestureContent += objectsText; + if (gesturesText) { + if (combinedObjectGestureContent) combinedObjectGestureContent += "\n\n"; + combinedObjectGestureContent += gesturesText; + } + if (!combinedObjectGestureContent) { + combinedObjectGestureContent = "**Objects & Gestures:**\n- None detected or analysis issue."; + } + setObjectCardContent(combinedObjectGestureContent); + } catch (err) { + console.error("MediaPipe analysis error:", err); + const errorMessage = err instanceof Error ? err.message : "Unknown error"; + setMediaPipeError(`Object detection failed: ${errorMessage}`); + } + // Run Groq vision analysis + setStatusMessage("Analyzing scene with Groq..."); try { - // Use the specific accessibility prompt when only image is captured. - const response = await getGroqVisionAnalysis(imageDataUrl, "You are an AI that helps blind people by describing what you see in an image.\nSpeak clearly and simply. Write your answer in first person, like you're talking to the user.\n\nStart by saying \u201CI see...\u201D\n\nDescribe the most important things in the image. For example: people, objects, actions, places.\n\nIf there is text in the image (like signs, books, screens), read it out loud in your answer.\n\nSpeak like a helpful friend. Use short sentences.\n\nOnly say what is clearly visible. Do not guess or imagine things.") - setAiResponse(response) - setStatusMessage(null) // Clear status on success - // Optionally speak the response - // speakText(response, () => setIsSpeaking(true), () => setIsSpeaking(false), (err) => console.error(err)); + const response = await getGroqVisionAnalysis(imageDataUrl, DETAILED_BASE_PROMPT); + setAiResponse(response); + + // Placeholder for other objects - Now handled by MediaPipe + // setObjectCardContent("**Detected Objects:**\n- **Desk Lamp** (On)\n- **Smartphone** (Screen off)\n\n**Gesture:**\n- Hand raised\n- Open palm"); } catch (err) { - console.error("Error analyzing image:", err) - const errorMessage = err instanceof Error ? err.message : "An unknown error occurred during analysis." - setError(`Analysis failed: ${errorMessage}`) - setStatusMessage(null) + console.error("Error analyzing image with Groq (voice query):", err); + const groqErrorMessage = err instanceof Error ? err.message : "An unknown error occurred during Groq analysis."; + setError(`Groq analysis failed: ${groqErrorMessage}`); + // setStatusMessage(null); // Status will be cleared in finally } finally { - setIsAnalyzing(false) + // setIsAnalyzing(false); // Moved down + // isSpeaking state is handled by TTS callbacks // No longer relevant here } - }, [captureFrame]) + + setIsAnalyzing(false); + setStatusMessage(null); // Clear status message + setShowCards(true); // Ensure cards are visible after analysis + }, [modelsReady, mediaPipeModelsReady, captureFrame, DETAILED_BASE_PROMPT, performFaceApiAnalysis, performMediaPipeAnalysis, getGroqVisionAnalysis, setAiResponse, setCapturedImagePreviewUrl, setObjectCardContent, setIdentityInfoContent, setError, setFaceApiError, setMediaPipeError, setShowCards, setStatusMessage]); // --- Flip Camera --- const handleFlipCamera = () => { @@ -151,6 +670,10 @@ export default function ARModePage() { // --- Microphone Toggle Logic --- const handleMicToggle = () => { + if (!modelsReady || !mediaPipeModelsReady) { // Check both + setError("AI models are not ready yet. Please wait or refresh."); + return; + } if (isMicListening) { stopMicListener(); } else { @@ -278,39 +801,76 @@ export default function ARModePage() { // --- Analysis Function (Voice Input) --- const handleAnalyzeVoice = useCallback(async (transcribedQuery: string) => { - setStatusMessage("Capturing image...") - setIsAnalyzing(true) - setAiResponse(null) - setError(null) - const imageDataUrl = captureFrame() + if (!modelsReady || !mediaPipeModelsReady) { // Check both + setError("AI models are not ready yet. Please wait or refresh."); + return; + } + setStatusMessage("Processing your request..."); + setIsAnalyzing(true); + setAiResponse(null); + setCapturedImagePreviewUrl(null); + setObjectCardContent(null); + setIdentityInfoContent(null); // Clear previous identity info for new analysis + setError(null); + setFaceApiError(null); + setMediaPipeError(null); // Clear MediaPipe error + const imageDataUrl = captureFrame(); // Capture frame first if (!imageDataUrl) { - setIsAnalyzing(false) - setStatusMessage(null) - setError("Could not capture frame for analysis.") - return + setIsAnalyzing(false); + setStatusMessage(null); + setError("Could not capture frame for analysis."); + return; } + setCapturedImagePreviewUrl(imageDataUrl); - setStatusMessage("Analyzing image with your query...") - console.log(`Frame captured, analyzing with query: \"${transcribedQuery}\"`) + // Perform Face-API analysis on the current video frame + setStatusMessage("Analyzing faces..."); + await performFaceApiAnalysis(videoRef.current); // Updates identityInfoContent - // Define the base accessibility prompt (same as in handleAnalyzeImage) - const basePrompt = "You are an AI that helps blind people by describing what you see in an image.\nSpeak clearly and simply. Write your answer in first person, like you're talking to the user.\n\nStart by saying \u201CI see...\u201D\n\nDescribe the most important things in the image. For example: people, objects, actions, places.\n\nIf there is text in the image (like signs, books, screens), read it out loud in your answer.\n\nSpeak like a helpful friend. Use short sentences.\n\nOnly say what is clearly visible. Do not guess or imagine things."; + // Perform MediaPipe Analysis + setStatusMessage("Analyzing objects & gestures..."); + const { objectsText, gesturesText } = await performMediaPipeAnalysis(videoRef.current); + + let combinedObjectGestureContent = ""; + if (objectsText) combinedObjectGestureContent += objectsText; + if (gesturesText) { + if (combinedObjectGestureContent) combinedObjectGestureContent += "\n\n"; + combinedObjectGestureContent += gesturesText; + } + if (!combinedObjectGestureContent) { + combinedObjectGestureContent = "**Objects & Gestures:**\n- None detected or analysis issue."; + } + setObjectCardContent(combinedObjectGestureContent); + + setStatusMessage("Analyzing scene with Groq..."); + console.log(`Frame captured, analyzing with Groq query: "${transcribedQuery}"`); - // Combine the base prompt with the user's query - const finalPrompt = `${basePrompt}\n\nBased on that description style, the user specifically asked: \"${transcribedQuery}\"` - console.log("Combined Vision Prompt:", finalPrompt); // Log the combined prompt + const finalPrompt = `${DETAILED_BASE_PROMPT}\n\nBased on that description style, the user specifically asked: "${transcribedQuery}"`; + console.log("Combined Vision Prompt for Groq:", finalPrompt); try { - // Send the combined prompt to the vision analysis function - const response = await getGroqVisionAnalysis(imageDataUrl, finalPrompt) - setAiResponse(response) - setStatusMessage("Speaking response...") - // Speak the response + const response = await getGroqVisionAnalysis(imageDataUrl, finalPrompt); + setAiResponse(response); + + // Placeholder for other objects - Now handled by MediaPipe + // setObjectCardContent("**Detected Objects:**\n- **Desk Lamp** (On)\n- **Smartphone** (Screen off)\n\n**Gesture:**\n- Hand raised\n- Open palm"); + + // TTS part + setStatusMessage("Speaking response..."); speakText( response, () => { console.log("TTS started"); setIsSpeaking(true); setStatusMessage("Speaking..."); }, - () => { console.log("TTS ended"); setIsSpeaking(false); setStatusMessage(null); setAiResponse(null); /* Auto-hide response */ }, + () => { + console.log("TTS ended"); + setIsSpeaking(false); + setStatusMessage(null); + // Auto-hide response and related cards after speaking + setAiResponse(null); + setCapturedImagePreviewUrl(null); + setObjectCardContent(null); + setIdentityInfoContent(null); + }, (err) => { console.error("TTS Error:", err); setError("Error speaking the response."); @@ -320,46 +880,123 @@ export default function ARModePage() { ); } catch (err) { - console.error("Error analyzing image with voice query:", err) - const errorMessage = err instanceof Error ? err.message : "An unknown error occurred during analysis." - setError(`Analysis failed: ${errorMessage}`) - setStatusMessage(null) + console.error("Error analyzing image with Groq (voice query):", err); + const groqErrorMessage = err instanceof Error ? err.message : "An unknown error occurred during Groq analysis."; + setError(`Groq analysis failed: ${groqErrorMessage}`); + setStatusMessage(null); } finally { - setIsAnalyzing(false) - // Don't set isSpeaking false here, TTS callback handles it + setIsAnalyzing(false); + // isSpeaking state is handled by TTS callbacks } - }, [captureFrame]) // Removed userQuery dependency + }, [captureFrame, modelsReady, mediaPipeModelsReady, DETAILED_BASE_PROMPT, objectDetector, gestureRecognizer]); + + // Clean up intervals on unmount + useEffect(() => { + return () => { + if (recordingIntervalRef.current) { + clearInterval(recordingIntervalRef.current); + console.log("Cleaned up recordingIntervalRef on unmount"); + } + if (captureIntervalRef.current) { + clearInterval(captureIntervalRef.current); + console.log("Cleaned up captureIntervalRef on unmount"); + } + }; + }, []); return ( -
- {/* Header */} -
- -

AR Mode

- {/* Placeholder for potential settings/flash toggle */} -
-
- - {/* Camera View */} -
{/* Ensure container has a background */} - {error ? ( -
- -

Error

{/* Simplified title */} -

{error}

- {/* Conditionally show Try Again button only for camera errors */} - {error.includes("camera") && - - } -
- ) : ( + // Conditionally render only on client to prevent hydration mismatches + isClient ? ( +
+ {/* Header */} +
+ +

AR Mode

+ {/* Recording Timer */} + {isRecording && ( +
+ + {formatRecordingTime(recordingTime)} +
+ )} + {!isRecording &&
} +
+ + {/* Camera View */} +
{/* Ensure container has a background */} + {(error || faceApiError || mediaPipeError) && ( // Display general error, faceApiError or mediaPipeError +
+ +

Error Encountered

+

{error || faceApiError || mediaPipeError}

+ {/* Show Try Again only for camera errors OR model loading errors */} + {( (error && error.includes("camera")) || faceApiError || mediaPipeError ) && + + } +
+ )} + {/* Video element must be rendered even if there's an error for setupCamera to attach to */}
+ )} +
+ +
+
+ {aiResponse} +
+
+ + )} + + + {/* Controls */} +
+ {/* Camera Toggle Button */} + + + {/* Camera Button */} + + + {/* Record Button (Updated) */} + + + {/* Voice Input Toggle Button */} + + +
+
+ ) : null ) } diff --git a/app/chat/[id]/page.tsx b/app/chat/[id]/page.tsx index 130823b..b863dbb 100644 --- a/app/chat/[id]/page.tsx +++ b/app/chat/[id]/page.tsx @@ -1009,11 +1009,7 @@ export default function ChatPage() { className={`flex ${message.role === "user" ? "justify-end" : "justify-start"} chat-bubble-in`} >
{/* --- Render Markdown for AI messages --- */} {message.role === 'assistant' ? ( diff --git a/app/layout.tsx b/app/layout.tsx index e4673b0..8146960 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -8,6 +8,8 @@ import { SpeedInsights } from "@vercel/speed-insights/next" import PageTransitionWrapper from "@/components/PageTransitionWrapper" import { SmoothCursor } from "@/components/ui/smooth-cursor" import { LenisProvider } from "@/context/LenisProvider" +import { StagewiseToolbar } from '@stagewise/toolbar-next' +import { Analytics } from "@vercel/analytics/next" const inter = Inter({ subsets: ["latin"], display: "swap" }) @@ -38,14 +40,16 @@ export default function RootLayout({ - + {process.env.NODE_ENV === 'development' && } + {/* */} {children} - + {/* */} + ) diff --git a/components/cta.tsx b/components/cta.tsx index c4e9e7e..4cc7991 100644 --- a/components/cta.tsx +++ b/components/cta.tsx @@ -2,14 +2,31 @@ import React from "react"; import { motion } from "framer-motion"; +import { LinkPreview } from "@/components/ui/link-preview"; export default function CTA() { return (
- Twitter - Linkedin - Github - Instagram + + Twitter + + + Linkedin + + + Github + + + Instagram +
); }; diff --git a/components/hero.tsx b/components/hero.tsx index 3d62aa8..ff17ae3 100644 --- a/components/hero.tsx +++ b/components/hero.tsx @@ -5,6 +5,7 @@ import Link from "next/link" import { motion } from "framer-motion" import { ArrowRight, Brain, Sparkles } from "lucide-react" import { useState, useEffect } from "react" +import { FlipWords } from "@/components/ui/flip-words"; interface ParticleStyle { width: number @@ -93,7 +94,7 @@ export default function Hero() { > - Introducing NbAIl - Now with AR Mode + Introducing NbAIl - Featuring Advanced Mixed Reality (MR) Integration
@@ -103,9 +104,9 @@ export default function Hero() { transition={{ duration: 0.5, delay: 0.1 }} className="text-4xl md:text-6xl lg:text-7xl font-bold text-white mb-6" > - Meet NbAIl — Your Multimodal{" "} + Meet NbAIl — Your AI that can{" "} - AI Assistant + @@ -129,22 +130,22 @@ export default function Hero() { diff --git a/components/ui/animated-tooltip.tsx b/components/ui/animated-tooltip.tsx new file mode 100644 index 0000000..d213dbd --- /dev/null +++ b/components/ui/animated-tooltip.tsx @@ -0,0 +1,104 @@ +"use client"; + +import React, { useState } from "react"; +import { + motion, + useTransform, + AnimatePresence, + useMotionValue, + useSpring, +} from "motion/react"; + +export const AnimatedTooltip = ({ + items, +}: { + items: { + id: number; + name: string; + designation: string; + image?: string; + textIcon?: string; + }[]; +}) => { + const [hoveredIndex, setHoveredIndex] = useState(null); + const springConfig = { stiffness: 100, damping: 5 }; + const x = useMotionValue(0); // going to set this value on mouse move + // rotate the tooltip + const rotate = useSpring( + useTransform(x, [-100, 100], [-45, 45]), + springConfig, + ); + // translate the tooltip + const translateX = useSpring( + useTransform(x, [-100, 100], [-50, 50]), + springConfig, + ); + const handleMouseMove = (event: any) => { + const halfWidth = event.target.offsetWidth / 2; + x.set(event.nativeEvent.offsetX - halfWidth); // set the x value, which is then used in transform and rotate + }; + + return ( + <> + {items.map((item, idx) => ( +
setHoveredIndex(item.id)} + onMouseLeave={() => setHoveredIndex(null)} + > + + {hoveredIndex === item.id && ( + +
+
+
+ {item.name} +
+
+ {item.designation} +
+ + )} + + {item.textIcon ? ( +
+ {item.textIcon} +
+ ) : item.image ? ( + {item.name} + ) : null} +
+ ))} + + ); +}; diff --git a/components/ui/flip-words.tsx b/components/ui/flip-words.tsx new file mode 100644 index 0000000..3a170c0 --- /dev/null +++ b/components/ui/flip-words.tsx @@ -0,0 +1,98 @@ +"use client"; +import React, { useCallback, useEffect, useRef, useState } from "react"; +import { AnimatePresence, motion, LayoutGroup } from "motion/react"; +import { cn } from "@/lib/utils"; + +export const FlipWords = ({ + words, + duration = 3000, + className, +}: { + words: string[]; + duration?: number; + className?: string; +}) => { + const [currentWord, setCurrentWord] = useState(words[0]); + const [isAnimating, setIsAnimating] = useState(false); + + // thanks for the fix Julian - https://github.com/Julian-AT + const startAnimation = useCallback(() => { + const word = words[words.indexOf(currentWord) + 1] || words[0]; + setCurrentWord(word); + setIsAnimating(true); + }, [currentWord, words]); + + useEffect(() => { + if (!isAnimating) + setTimeout(() => { + startAnimation(); + }, duration); + }, [isAnimating, duration, startAnimation]); + + return ( + { + setIsAnimating(false); + }} + > + + {/* edit suggested by Sajal: https://x.com/DewanganSajal */} + {currentWord.split(" ").map((word, wordIndex) => ( + + {word.split("").map((letter, letterIndex) => ( + + {letter} + + ))} +   + + ))} + + + ); +}; diff --git a/components/ui/following-pointer.tsx b/components/ui/following-pointer.tsx new file mode 100644 index 0000000..b294695 --- /dev/null +++ b/components/ui/following-pointer.tsx @@ -0,0 +1,136 @@ +import React, { useEffect, useState } from "react"; + +import { motion, AnimatePresence, useMotionValue } from "motion/react"; +import { cn } from "@/lib/utils"; + +export const FollowerPointerCard = ({ + children, + className, + title, +}: { + children: React.ReactNode; + className?: string; + title?: string | React.ReactNode; +}) => { + const x = useMotionValue(0); + const y = useMotionValue(0); + const ref = React.useRef(null); + const [rect, setRect] = useState(null); + const [isInside, setIsInside] = useState(false); // Add this line + + useEffect(() => { + if (ref.current) { + setRect(ref.current.getBoundingClientRect()); + } + }, []); + + const handleMouseMove = (e: React.MouseEvent) => { + if (rect) { + const scrollX = window.scrollX; + const scrollY = window.scrollY; + x.set(e.clientX - rect.left + scrollX); + y.set(e.clientY - rect.top + scrollY); + } + }; + const handleMouseLeave = () => { + setIsInside(false); + }; + + const handleMouseEnter = () => { + setIsInside(true); + }; + return ( +
+ + {isInside && } + + {children} +
+ ); +}; + +export const FollowPointer = ({ + x, + y, + title, +}: { + x: any; + y: any; + title?: string | React.ReactNode; +}) => { + const colors = [ + "#0ea5e9", + "#737373", + "#14b8a6", + "#22c55e", + "#3b82f6", + "#ef4444", + "#eab308", + ]; + return ( + + + + + + {title || `William Shakespeare`} + + + ); +}; diff --git a/components/ui/link-preview.tsx b/components/ui/link-preview.tsx new file mode 100644 index 0000000..d1be833 --- /dev/null +++ b/components/ui/link-preview.tsx @@ -0,0 +1,152 @@ + +"use client"; +import * as HoverCardPrimitive from "@radix-ui/react-hover-card"; + +import { encode } from "qss"; +import React from "react"; +import { + AnimatePresence, + motion, + useMotionValue, + useSpring, +} from "motion/react"; + +import { cn } from "@/lib/utils"; + +type LinkPreviewProps = { + children: React.ReactNode; + url: string; + className?: string; + width?: number; + height?: number; + quality?: number; + layout?: string; +} & ( + | { isStatic: true; imageSrc: string } + | { isStatic?: false; imageSrc?: never } +); + +export const LinkPreview = ({ + children, + url, + className, + width = 200, + height = 125, + quality = 50, + layout = "fixed", + isStatic = false, + imageSrc = "", +}: LinkPreviewProps) => { + let src; + if (!isStatic) { + const params = encode({ + url, + screenshot: true, + meta: false, + embed: "screenshot.url", + colorScheme: "dark", + "viewport.isMobile": true, + "viewport.deviceScaleFactor": 1, + "viewport.width": width * 3, + "viewport.height": height * 3, + }); + src = `https://api.microlink.io/?${params}`; + } else { + src = imageSrc; + } + + const [isOpen, setOpen] = React.useState(false); + + const [isMounted, setIsMounted] = React.useState(false); + + React.useEffect(() => { + setIsMounted(true); + }, []); + + const springConfig = { stiffness: 100, damping: 15 }; + const x = useMotionValue(0); + + const translateX = useSpring(x, springConfig); + + const handleMouseMove = (event: any) => { + const targetRect = event.target.getBoundingClientRect(); + const eventOffsetX = event.clientX - targetRect.left; + const offsetFromCenter = (eventOffsetX - targetRect.width / 2) / 2; // Reduce the effect to make it subtle + x.set(offsetFromCenter); + }; + + return ( + <> + {isMounted ? ( +
+ hidden image +
+ ) : null} + + { + setOpen(open); + }} + > + + {children} + + + + + {isOpen && ( + + + preview image + + + )} + + + + + ); +}; diff --git a/faceapi-docs.md b/faceapi-docs.md new file mode 100644 index 0000000..df98850 --- /dev/null +++ b/faceapi-docs.md @@ -0,0 +1,855 @@ +# face-api.js + +[![Build Status](https://travis-ci.org/justadudewhohacks/face-api.js.svg?branch=master)](https://travis-ci.org/justadudewhohacks/face-api.js) +[![Slack](https://slack.bri.im/badge.svg)](https://slack.bri.im) + +**JavaScript face recognition API for the browser and nodejs implemented on top of tensorflow.js core ([tensorflow/tfjs-core](https://github.com/tensorflow/tfjs))** + +![faceapi](https://user-images.githubusercontent.com/31125521/57224752-ad3dc080-700a-11e9-85b9-1357b9f9bca4.gif) + +## **[Click me for Live Demos!](https://justadudewhohacks.github.io/face-api.js/)** + +## Tutorials + +* **[face-api.js — JavaScript API for Face Recognition in the Browser with tensorflow.js](https://itnext.io/face-api-js-javascript-api-for-face-recognition-in-the-browser-with-tensorflow-js-bcc2a6c4cf07)** +* **[Realtime JavaScript Face Tracking and Face Recognition using face-api.js’ MTCNN Face Detector](https://itnext.io/realtime-javascript-face-tracking-and-face-recognition-using-face-api-js-mtcnn-face-detector-d924dd8b5740)** +* **[Realtime Webcam Face Detection And Emotion Recognition - Video](https://youtu.be/CVClHLwv-4I)** +* **[Easy Face Recognition Tutorial With JavaScript - Video](https://youtu.be/AZ4PdALMqx0)** +* **[Using face-api.js with Vue.js and Electron](https://medium.com/@andreas.schallwig/do-not-laugh-a-simple-ai-powered-game-3e22ad0f8166)** +* **[Add Masks to People - Gant Laborde on Learn with Jason](https://www.learnwithjason.dev/fun-with-machine-learning-pt-2)** + +## Table of Contents + +* **[Features](#features)** +* **[Running the Examples](#running-the-examples)** +* **[face-api.js for the Browser](#face-api.js-for-the-browser)** +* **[face-api.js for Nodejs](#face-api.js-for-nodejs)** +* **[Usage](#getting-started)** + * **[Loading the Models](#getting-started-loading-models)** + * **[High Level API](#high-level-api)** + * **[Displaying Detection Results](#getting-started-displaying-detection-results)** + * **[Face Detection Options](#getting-started-face-detection-options)** + * **[Utility Classes](#getting-started-utility-classes)** + * **[Other Useful Utility](#other-useful-utility)** +* **[Available Models](#models)** + * **[Face Detection](#models-face-detection)** + * **[Face Landmark Detection](#models-face-landmark-detection)** + * **[Face Recognition](#models-face-recognition)** + * **[Face Expression Recognition](#models-face-expression-recognition)** + * **[Age Estimation and Gender Recognition](#models-age-and-gender-recognition)** +* **[API Documentation](https://justadudewhohacks.github.io/face-api.js/docs/globals.html)** + +# Features + +## Face Recognition + +![face-recognition](https://user-images.githubusercontent.com/31125521/57297377-bfcdfd80-70cf-11e9-8afa-2044cb167bff.gif) + +## Face Landmark Detection + +![face_landmark_detection](https://user-images.githubusercontent.com/31125521/57297731-b1ccac80-70d0-11e9-9bd7-59d77f180322.jpg) + +## Face Expression Recognition + +![preview_face-expression-recognition](https://user-images.githubusercontent.com/31125521/50575270-f501d080-0dfb-11e9-9676-8f419efdade4.png) + +## Age Estimation & Gender Recognition + +![age_gender_recognition](https://user-images.githubusercontent.com/31125521/57297736-b5603380-70d0-11e9-873d-8b6c7243eb64.jpg) + + + +# Running the Examples + +Clone the repository: + +``` bash +git clone https://github.com/justadudewhohacks/face-api.js.git +``` + +## Running the Browser Examples + +``` bash +cd face-api.js/examples/examples-browser +npm i +npm start +``` + +Browse to http://localhost:3000/. + +## Running the Nodejs Examples + +``` bash +cd face-api.js/examples/examples-nodejs +npm i +``` + +Now run one of the examples using ts-node: + +``` bash +ts-node faceDetection.ts +``` + +Or simply compile and run them with node: + +``` bash +tsc faceDetection.ts +node faceDetection.js +``` + + + +# face-api.js for the Browser + +Simply include the latest script from [dist/face-api.js](https://github.com/justadudewhohacks/face-api.js/tree/master/dist). + +Or install it via npm: + +``` bash +npm i face-api.js +``` + + + +# face-api.js for Nodejs + +We can use the equivalent API in a nodejs environment by polyfilling some browser specifics, such as HTMLImageElement, HTMLCanvasElement and ImageData. The easiest way to do so is by installing the node-canvas package. + +Alternatively you can simply construct your own tensors from image data and pass tensors as inputs to the API. + +Furthermore you want to install @tensorflow/tfjs-node (not required, but highly recommended), which speeds things up drastically by compiling and binding to the native Tensorflow C++ library: + +``` bash +npm i face-api.js canvas @tensorflow/tfjs-node +``` + +Now we simply monkey patch the environment to use the polyfills: + +``` javascript +// import nodejs bindings to native tensorflow, +// not required, but will speed up things drastically (python required) +import '@tensorflow/tfjs-node'; + +// implements nodejs wrappers for HTMLCanvasElement, HTMLImageElement, ImageData +import * as canvas from 'canvas'; + +import * as faceapi from 'face-api.js'; + +// patch nodejs environment, we need to provide an implementation of +// HTMLCanvasElement and HTMLImageElement +const { Canvas, Image, ImageData } = canvas +faceapi.env.monkeyPatch({ Canvas, Image, ImageData }) +``` + + + +# Getting Started + + + +## Loading the Models + +All global neural network instances are exported via faceapi.nets: + +``` javascript +console.log(faceapi.nets) +// ageGenderNet +// faceExpressionNet +// faceLandmark68Net +// faceLandmark68TinyNet +// faceRecognitionNet +// ssdMobilenetv1 +// tinyFaceDetector +// tinyYolov2 +``` + +To load a model, you have to provide the corresponding manifest.json file as well as the model weight files (shards) as assets. Simply copy them to your public or assets folder. The manifest.json and shard files of a model have to be located in the same directory / accessible under the same route. + +Assuming the models reside in **public/models**: + +``` javascript +await faceapi.nets.ssdMobilenetv1.loadFromUri('/models') +// accordingly for the other models: +// await faceapi.nets.faceLandmark68Net.loadFromUri('/models') +// await faceapi.nets.faceRecognitionNet.loadFromUri('/models') +// ... +``` + +In a nodejs environment you can furthermore load the models directly from disk: + +``` javascript +await faceapi.nets.ssdMobilenetv1.loadFromDisk('./models') +``` + +You can also load the model from a tf.NamedTensorMap: + +``` javascript +await faceapi.nets.ssdMobilenetv1.loadFromWeightMap(weightMap) +``` + +Alternatively, you can also create own instances of the neural nets: + +``` javascript +const net = new faceapi.SsdMobilenetv1() +await net.loadFromUri('/models') +``` + +You can also load the weights as a Float32Array (in case you want to use the uncompressed models): + +``` javascript +// using fetch +net.load(await faceapi.fetchNetWeights('/models/face_detection_model.weights')) + +// using axios +const res = await axios.get('/models/face_detection_model.weights', { responseType: 'arraybuffer' }) +const weights = new Float32Array(res.data) +net.load(weights) +``` + + + +## High Level API + +In the following **input** can be an HTML img, video or canvas element or the id of that element. + +``` html + +
- - - {/* Camera View */} -
{/* Ensure container has a background */} - {(error || faceApiError || mediaPipeError) && ( // Display general error, faceApiError or mediaPipeError -
- -

Error Encountered

-

{error || faceApiError || mediaPipeError}

- {/* Show Try Again only for camera errors OR model loading errors */} - {( (error && error.includes("camera")) || faceApiError || mediaPipeError ) && - - } -
- )} - {/* Video element must be rendered even if there's an error for setupCamera to attach to */} -
- - {/* Top-Left Captured Image Preview Card - Now includes identity info */} - - {showCards && (capturedImagePreviewUrl || identityInfoContent) && ( // Show if either image OR identity info is present - - {capturedImagePreviewUrl && - Captured scene - } - {identityInfoContent && ( -
- {identityInfoContent} -
- )} - - )} - - - {/* Object Card (Bottom-Left) */} - - {showCards && objectCardContent && ( - -
- {objectCardContent} -
- - )} - - - - {/* AI Response Overlay (Bottom-Right Scene Analysis) */} - - {showCards && aiResponse && ( - -
-

Scene Analysis:

- {/* Hide close button during recording for AI response card to prevent accidental closure */} - {!isRecording && ( - + {/* Info Widget with updated styling */} +
+ {/* Assuming InfoWidget only provides content, not wrapping div */} +
+
+ + + {/* Camera View */} +
{/* Ensure container has a background */} + {(error || faceApiError || mediaPipeError) && ( // Display general error, faceApiError or mediaPipeError +
+ +

Error Encountered

+

{error || faceApiError || mediaPipeError}

+ {/* Show Try Again only for camera errors OR model loading errors */} + {( (error && error.includes("camera")) || faceApiError || mediaPipeError ) && + + } +
+ )} + {/* Video element must be rendered even if there's an error for setupCamera to attach to */} +
+ + {/* Cards Container - Positioned below camera or as overlay */} +
+
+ {/* Detected Person Card */} + + {showCards && (capturedImagePreviewUrl || identityInfoContent) && ( + +

Detected Person:

+ {capturedImagePreviewUrl && + Captured scene + } + {identityInfoContent && ( +
+ {identityInfoContent} +
+ )} + )} + + + {/* Detected Objects/Gestures Card */} + + {showCards && objectCardContent && ( + +

Detected Objects/Gestures:

+
+ {objectCardContent} +
+ + )} + + + {/* Scene Analysis Card */} + + {showCards && aiResponse && ( + +
+

Scene Analysis:

+ {/* Hide close button during recording for AI response card to prevent accidental closure */} + {!isRecording && ( + + )} +
+
+ {aiResponse} +
+ + )} +
-
- {aiResponse} -
- - )} - - - {/* Controls */} -
- {/* Camera Toggle Button */} - - - {/* Camera Button (Analyze Image) */} - - - {/* Record Button */} - - - {/* Voice Input Toggle Button */} - +
- -
- ) + {/* Controls */} +
+ {/* Camera Toggle Button */} + + + {/* Camera Button (Analyze Image) */} + + + {/* Record Button */} + + + {/* Voice Input Toggle Button */} + + +
+
+ ) } \ No newline at end of file diff --git a/app/chat/[id]/page.tsx b/app/chat/[id]/page.tsx index b863dbb..1cbe04c 100644 --- a/app/chat/[id]/page.tsx +++ b/app/chat/[id]/page.tsx @@ -1151,32 +1151,11 @@ export default function ChatPage() { )} {/* Input area & Image Preview */} -
- {/* Image Preview Section */} - {imageBase64 && ( -
-
- Selected preview - -
-
- )} - {/* End Image Preview Section */} - +
{/* Main input container - ChatGPT style rounded pill */} -
+
{/* Left side buttons */}
@@ -1230,7 +1209,7 @@ export default function ChatPage() {
{/* Input field */} -
+
void; +} + +const Switch: React.FC = ({ isToggled, onToggle }) => { + return ( + +
+ +