diff --git a/.gitignore b/.gitignore index 6470c2d..9eb81fe 100644 --- a/.gitignore +++ b/.gitignore @@ -91,4 +91,6 @@ temp/ logs/ -aura/cnn/checkpoints/ \ No newline at end of file +aura/cnn/checkpoints/ + +*.pth \ No newline at end of file diff --git a/aura/webrtc/signaling.py b/aura/webrtc/signaling.py index bcd56c9..920f451 100644 --- a/aura/webrtc/signaling.py +++ b/aura/webrtc/signaling.py @@ -4,7 +4,7 @@ import socket import signal import time -from camera import ProcessingPipeline, FaceNotFoundException +from aura.camera import ProcessingPipeline, FaceNotFoundException import os import numpy as np import cv2 @@ -14,39 +14,38 @@ def get_free_port(): """Get an unused TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(('', 0)) + s.bind(("", 0)) return s.getsockname()[1] + def signal_handler(sig, frame): """Handle Ctrl+C gracefully""" print("\nShutting down signaling server...") sys.exit(0) + def capture_images(server, output_dir="../logs", verbose=2): """Capture, process, and save images with face detection""" os.makedirs(output_dir, exist_ok=True) - - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - - pipeline = ProcessingPipeline( - log_path=output_dir, - verbose=verbose - ) - + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + pipeline = ProcessingPipeline(log_path=output_dir, verbose=verbose) + server.capture() time.sleep(2) - + image_bytes = server.get_capture() if image_bytes: nparr = np.frombuffer(image_bytes, np.uint8) image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) - - try: + + try: annotated_image = pipeline.annotate_face(image) processed_face = pipeline.process_image(image) - - # TODO: Send processed face to the model - + + # TODO: Send processed face to the model + except FaceNotFoundException: print(f"No face detected in captured image.") except Exception as e: @@ -54,17 +53,18 @@ def capture_images(server, output_dir="../logs", verbose=2): else: print("No image captured") + def main(): signal.signal(signal.SIGINT, signal_handler) - port = 8765 - + port = 3030 + server = SignalingServer(port=port) server.start() - + print(f"Signaling server started on port {port}") print("Press Ctrl+C to stop the server") - + try: while True: time.sleep(10) @@ -73,7 +73,6 @@ def main(): except KeyboardInterrupt: print("\nShutting down signaling server...") + if __name__ == "__main__": - help(SignalingServer) - help(VideoStreamer) - \ No newline at end of file + main() diff --git a/aura/webrtc/streamer.py b/aura/webrtc/streamer.py index 2f39b5f..5edb703 100644 --- a/aura/webrtc/streamer.py +++ b/aura/webrtc/streamer.py @@ -4,31 +4,33 @@ import time from aura import VideoStreamer + def signal_handler(sig, frame): """Handle Ctrl+C gracefully""" print("\nShutting down video streamer...") sys.exit(0) + def main(): signal.signal(signal.SIGINT, signal_handler) ws_ip = "127.0.0.1" # WebSocket IP address - ws_port = 8765 # WebSocket port - ivf_dir = "../ivf_dir" # Directory to watch for IVF files + ws_port = 3030 # WebSocket port + ivf_dir = "./ivf_files" # Match RustWebRTC directory structure streamer = VideoStreamer(ws_ip, ws_port, ivf_dir) streamer.start_streaming() - + print(f"Video streamer started - WebSocket server at ws://{ws_ip}:{ws_port}") print(f"Watching directory: {ivf_dir}") print("Press Ctrl+C to stop the streamer") - + try: while True: time.sleep(1) except KeyboardInterrupt: print("\nShutting down video streamer...") + if __name__ == "__main__": main() - diff --git a/scripts/convert_to_ivf.sh b/scripts/convert_to_ivf.sh index 85a19aa..50b992c 100755 --- a/scripts/convert_to_ivf.sh +++ b/scripts/convert_to_ivf.sh @@ -1,5 +1,6 @@ #!/bin/bash +# Check if input file is provided if [ $# -ne 1 ]; then echo "Usage: $0 input.mp4" exit 1 @@ -8,11 +9,13 @@ fi input_file="$1" output_file="${input_file%.*}.ivf" +# Check if input file exists if [ ! -f "$input_file" ]; then echo "Error: Input file '$input_file' not found" exit 1 fi +# Convert to IVF using VP8 codec ffmpeg -i "$input_file" -c:v libvpx -an -f ivf "$output_file" if [ $? -eq 0 ]; then diff --git a/src/server.rs b/src/server.rs index 07f608c..044b494 100644 --- a/src/server.rs +++ b/src/server.rs @@ -100,6 +100,7 @@ impl SignalingServer { Ok(()) } + #[pyo3(text_signature = "(self, client_id: str, message: str) -> bool")] pub fn send_to_client(&self, client_id: String, message: String) -> PyResult { let peers = self.peers.clone(); @@ -243,9 +244,14 @@ async fn handle_connection( match result { Ok(msg) => { if let Ok(text) = msg.to_str() { + println!("Received message from {}: {}", client_id, text); + + // Attempt to parse the message let signaling_message: Result = serde_json::from_str(text); match signaling_message { Ok(SignalingMessage::Image { data }) => { + // Handle image message + println!("Handling image message from client {}", client_id); handle_image_message(data.clone()).await; if let Some(base64_data) = data.split(',').nth(1) { @@ -258,16 +264,21 @@ async fn handle_connection( } } Ok(message) => { + // Handle other signaling messages + println!("Parsed signaling message: {:?}", message); forward_message(&client_id, &message, &peers).await; } Err(e) => { - eprintln!("Error parsing message: {:?}", e); + eprintln!( + "Error parsing message from client {}: {} - Error: {:?}", + client_id, text, e + ); } } } } Err(e) => { - eprintln!("Error receiving message: {}", e); + eprintln!("Error receiving message for client {}: {}", client_id, e); break; } } @@ -302,6 +313,16 @@ async fn handle_image_message(data: String) { } } +async fn trigger_image_capture( + sender: Arc>>, +) -> Result<(), Box> { + let message = serde_json::to_string(&SignalingMessage::TriggerImageCapture)?; + let mut sender = sender.lock().await; + sender.send(Message::text(message)).await?; + println!("Sent image capture trigger to client."); + Ok(()) +} + async fn forward_message(sender_id: &str, message: &SignalingMessage, peers: &Peers) { let serialized_message = match serde_json::to_string(message) { Ok(json) => json, @@ -311,7 +332,7 @@ async fn forward_message(sender_id: &str, message: &SignalingMessage, peers: &Pe } }; - let peers = peers.lock().await; + let peers = peers.lock().await; // Await the async Mutex lock for (client_id, client) in peers.iter() { if client_id != sender_id { let mut client = client.lock().await; // Await the async Mutex lock @@ -320,14 +341,4 @@ async fn forward_message(sender_id: &str, message: &SignalingMessage, peers: &Pe } } } -} - -async fn trigger_image_capture( - sender: Arc>>, -) -> Result<(), Box> { - let message = serde_json::to_string(&SignalingMessage::TriggerImageCapture)?; - let mut sender = sender.lock().await; - sender.send(Message::text(message)).await?; - println!("Sent image capture trigger to client."); - Ok(()) -} +} \ No newline at end of file diff --git a/src/streamer.rs b/src/streamer.rs index 05a5ac7..60d3c7b 100644 --- a/src/streamer.rs +++ b/src/streamer.rs @@ -9,9 +9,7 @@ use std::sync::Arc; use std::{fs::File, io::BufReader, time::Duration}; use tokio::sync::mpsc; use tokio::sync::Mutex; -use tokio_tungstenite::{connect_async, tungstenite::Message as TungsteniteMessage}; -use webrtc::peer_connection::RTCPeerConnection; -use webrtc::rtp_transceiver::rtp_codec::RTPCodecType; +use tokio_tungstenite::{connect_async, tungstenite::Message}; use webrtc::{ api::{ interceptor_registry::register_default_interceptors, @@ -34,7 +32,6 @@ pub struct VideoStreamer { ws_ip: String, ws_port: u16, ivf_dir: String, - peer_connection: Arc>>>, } #[pymethods] @@ -46,7 +43,6 @@ impl VideoStreamer { ws_ip, ws_port, ivf_dir, - peer_connection: Arc::new(Mutex::new(None)), } } @@ -55,14 +51,11 @@ impl VideoStreamer { let ws_ip = self.ws_ip.clone(); let ws_port = self.ws_port; let ivf_dir = self.ivf_dir.clone(); - let peer_connection_store = self.peer_connection.clone(); std::thread::spawn(move || { let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async move { - // Setup WebRTC and signaling - if let Err(e) = start_webrtc(&ws_ip, ws_port, &ivf_dir, peer_connection_store).await - { + if let Err(e) = start_webrtc(&ws_ip, ws_port, &ivf_dir).await { eprintln!("Error starting WebRTC: {}", e); } }); @@ -70,124 +63,24 @@ impl VideoStreamer { Ok(()) } - - #[pyo3(text_signature = "(self) -> bytes")] - fn take_screenshot(&self) -> PyResult> { - let peer_connection = self.peer_connection.clone(); - - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async move { - if let Some(pc) = peer_connection.lock().await.as_ref() { - match capture_screenshot(pc).await { - Ok(screenshot) => Ok(screenshot), - Err(e) => Err(PyErr::new::(format!( - "Failed to capture screenshot: {}", - e - ))), - } - } else { - Err(PyErr::new::( - "No active peer connection", - )) - } - }) - } - - #[pyo3(text_signature = "(self) -> str")] - fn get_connection_state(&self) -> PyResult { - let peer_connection = self.peer_connection.clone(); - - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async move { - if let Some(pc) = peer_connection.lock().await.as_ref() { - Ok(pc.connection_state().to_string()) - } else { - Err(PyErr::new::( - "No active peer connection", - )) - } - }) - } - - #[pyo3(text_signature = "(self) -> str")] - fn get_signaling_state(&self) -> PyResult { - let peer_connection = self.peer_connection.clone(); - - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async move { - if let Some(pc) = peer_connection.lock().await.as_ref() { - Ok(pc.signaling_state().to_string()) - } else { - Err(PyErr::new::( - "No active peer connection", - )) - } - }) - } - - #[pyo3(text_signature = "(self) -> str")] - fn get_stats(&self) -> PyResult { - let peer_connection = self.peer_connection.clone(); - - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async move { - if let Some(pc) = peer_connection.lock().await.as_ref() { - let stats = pc.get_stats().await; - match serde_json::to_string(&stats) { - Ok(stats_string) => Ok(stats_string), - Err(e) => Err(PyErr::new::(format!( - "Failed to serialize stats: {}", - e - ))), - } - } else { - Err(PyErr::new::( - "No active peer connection", - )) - } - }) - } - - #[pyo3(text_signature = "(self) -> None")] - fn close_connection(&self) -> PyResult<()> { - let peer_connection = self.peer_connection.clone(); - let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async move { - if let Some(pc) = peer_connection.lock().await.as_ref() { - if let Err(e) = pc.close().await { - return Err(PyErr::new::(format!( - "Failed to close connection: {}", - e - ))); - } - *peer_connection.lock().await = None; - Ok(()) - } else { - Err(PyErr::new::( - "No active peer connection", - )) - } - }) - } } -async fn start_webrtc( - ws_ip: &str, - ws_port: u16, - ivf_dir: &str, - peer_connection_store: Arc>>>, -) -> Result<()> { +async fn start_webrtc(ws_ip: &str, ws_port: u16, ivf_dir: &str) -> Result<()> { + // Create MediaEngine let mut m = MediaEngine::default(); m.register_default_codecs()?; + // Create a registry for interceptors let mut registry = Registry::new(); registry = register_default_interceptors(registry, &mut m)?; + // Create the API object let api = APIBuilder::new() .with_media_engine(m) .with_interceptor_registry(registry) .build(); + // Prepare the configuration let config = RTCConfiguration { ice_servers: vec![RTCIceServer { urls: vec!["stun:stun.l.google.com:19302".to_owned()], @@ -196,8 +89,10 @@ async fn start_webrtc( ..Default::default() }; + // Create a new RTCPeerConnection let peer_connection = Arc::new(api.new_peer_connection(config).await?); - *peer_connection_store.lock().await = Some(Arc::clone(&peer_connection)); + + // Create video track let video_track = Arc::new(TrackLocalStaticSample::new( RTCRtpCodecCapability { mime_type: MIME_TYPE_VP8.to_owned(), @@ -207,10 +102,12 @@ async fn start_webrtc( "webcam".to_owned(), )); + // Add track to peer connection let rtp_sender = peer_connection .add_track(Arc::clone(&video_track) as Arc) .await?; + // Handle RTCP packets tokio::spawn(async move { let mut rtcp_buf = vec![0u8; 1500]; while let Ok((_, _)) = rtp_sender.read(&mut rtcp_buf).await {} @@ -218,15 +115,17 @@ async fn start_webrtc( // Connect to signaling server let (ws_stream, _) = connect_async(format!("ws://{}:{}/signaling", ws_ip, ws_port)).await?; - let (write, mut read) = ws_stream.split(); + let (mut write, mut read) = ws_stream.split(); let write = Arc::new(Mutex::new(write)); let pc = Arc::clone(&peer_connection); + // Handle connection state changes peer_connection.on_peer_connection_state_change(Box::new(move |s: RTCPeerConnectionState| { println!("Connection State has changed: {s}"); Box::pin(async {}) })); + // Handle incoming messages let write_clone = Arc::clone(&write); tokio::spawn(async move { while let Some(msg) = read.next().await { @@ -244,9 +143,7 @@ async fn start_webrtc( let msg = SignalingMessage::Answer { sdp: answer.sdp }; let mut write = write_clone.lock().await; write - .send(TungsteniteMessage::Text( - serde_json::to_string(&msg).unwrap(), - )) + .send(Message::Text(serde_json::to_string(&msg).unwrap())) .await .unwrap(); } @@ -273,9 +170,7 @@ async fn start_webrtc( println!("Received image message - ignoring in WebRTC context"); } SignalingMessage::TriggerImageCapture => { - println!( - "Received trigger capture message - ignoring in WebRTC context" - ); + println!("Received trigger capture message - ignoring in WebRTC context"); } } } @@ -284,7 +179,7 @@ async fn start_webrtc( }); println!("Starting video stream..."); - watch_and_stream_video(ivf_dir, video_track).await?; + watchand_stream_video(ivf_dir, video_track).await?; Ok(()) } @@ -312,45 +207,12 @@ async fn write_video_to_track(path: &str, track: Arc) -> } } -async fn capture_screenshot(peer_connection: &Arc) -> Result> { - let transceivers = peer_connection.get_transceivers().await; - - for transceiver in transceivers { - let receiver = transceiver.receiver().await; - let tracks = receiver.tracks().await; - - for track in tracks { - if track.kind() == RTPCodecType::Video { - let mut buffer = vec![0u8; 1500]; - - let (tx, mut rx) = mpsc::channel::>(1); - let tx = tx.clone(); - - tokio::spawn(async move { - if let Ok((rtp_packet, _)) = track.read(&mut buffer).await { - // Access the payload data from the RTP packet - let payload = rtp_packet.payload.clone(); - let _ = tx.send(payload.to_vec()).await; - } - }); - - if let Ok(Some(frame_data)) = - tokio::time::timeout(Duration::from_secs(5), rx.recv()).await - { - return Ok(frame_data); - } - } - } - } - - Err(anyhow::Error::msg( - "No video track found or timeout occurred", - )) -} - -async fn watch_and_stream_video(directory: &str, track: Arc) -> Result<()> { +//File watcher +async fn watchand_stream_video(directory: &str, track: Arc) -> Result<()> { + // Create a channel for file events let (tx, mut rx) = mpsc::channel(100); + // Create an async file watcher let mut watcher = RecommendedWatcher::new( move |res| { if let Ok(event) = res { @@ -360,12 +222,14 @@ async fn watch_and_stream_video(directory: &str, track: Arc { } } else if (data.type === "image") { console.log("Received image data:", data.data); + } else if (data.type === "triggerimagecapture") { + console.log("Received trigger from server, capturing frame..."); + captureFrame(); + } else { + console.log("Unknown message type:", data); } }; @@ -112,20 +117,9 @@ pc.ontrack = (event) => { } }; -signalingSocket.onmessage = (event) => { - const message = JSON.parse(event.data); - if (message.type === "triggerimagecapture") { - console.log("Received trigger from server, capturing frame..."); - captureFrame(); - } else { - console.log("Unknown message type:", message); - } - }; - const captureImageBtn = document.getElementById("captureImage"); const capturedImage = document.getElementById("capturedImage"); - // Capture the video frame and display it as an image captureImageBtn.addEventListener("click", () => { // Create a canvas element dynamically @@ -177,6 +171,4 @@ function captureFrame() { } else { console.error("Failed to get canvas context for drawing."); } -} - - +} \ No newline at end of file diff --git a/ui/static/index.html b/ui/static/index.html index 0524f2d..3728fec 100644 --- a/ui/static/index.html +++ b/ui/static/index.html @@ -1,23 +1,22 @@ - - - + + + Rust WebRTC.rs Application - - + +

Rust WebRTC.rs Application

@@ -30,5 +29,5 @@

Rust WebRTC.rs Application

Captured Frame - - + + \ No newline at end of file