From 86b637c0d6d9d91567135d89a00b8839c66b4fd6 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 24 Mar 2026 10:40:54 +0000 Subject: [PATCH 01/65] Replace MoviePy with ffmpeg for massive performance improvement - Parallel downloads (4 concurrent) instead of sequential - Increase download chunk size from 8KB to 1MB - Replace MoviePy video processing with direct ffmpeg calls - Use ffmpeg concat demuxer with -c copy (no re-encoding) - Normalize segments to common resolution for reliable concat - Handle gaps with lightweight ffmpeg-generated black segments - Merge audio-only tracks using ffmpeg filter_complex - Remove moviepy and numpy dependencies - Add ffmpeg to Dockerfile - Bump version to 2.0.0 Fixes #8 --- Dockerfile | 14 +- mtslinker/__init__.py | 2 +- mtslinker/downloader.py | 46 +++++- mtslinker/processor.py | 347 +++++++++++++++++++++++++++++++--------- mtslinker/webinar.py | 23 ++- requirements.txt | 1 - setup.py | 3 +- 7 files changed, 337 insertions(+), 99 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1c0ad66..e19049c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,22 @@ FROM python:3.13-slim WORKDIR /app -# Копируем только requirements.txt сначала для кэширования +# Install ffmpeg +RUN apt-get update && \ + apt-get install -y --no-install-recommends ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +# Copy requirements first for caching COPY requirements.txt . -# Устанавливаем зависимости +# Install dependencies RUN pip install --no-cache-dir --upgrade pip && \ pip install --no-cache-dir -r requirements.txt -# Копируем остальные файлы проекта +# Copy project files COPY . . -# Устанавливаем пакет (если он setup как пакет) +# Install package RUN if [ -f "setup.py" ]; then pip install --no-cache-dir .; fi ENTRYPOINT ["mtslinker"] - diff --git a/mtslinker/__init__.py b/mtslinker/__init__.py index 6f621ca..1518e9f 100644 --- a/mtslinker/__init__.py +++ b/mtslinker/__init__.py @@ -1,3 +1,3 @@ from mtslinker.utils import initialize_logger -initialize_logger() \ No newline at end of file +initialize_logger() diff --git a/mtslinker/downloader.py b/mtslinker/downloader.py index 99f886a..412e001 100644 --- a/mtslinker/downloader.py +++ b/mtslinker/downloader.py @@ -1,17 +1,20 @@ import os -from typing import Dict, Union +from typing import Dict, List, Tuple, Union +from concurrent.futures import ThreadPoolExecutor, as_completed import httpx import tqdm import logging TIMEOUT_SETTINGS = httpx.Timeout(None, connect=None) +CHUNK_SIZE = 1024 * 1024 # 1 MB +MAX_PARALLEL_DOWNLOADS = 4 def construct_json_data_url(event_session_id: str, recording_id: str) -> str: if not event_session_id: raise ValueError('Missing webinar event session ID.') - + if not recording_id: return f'https://my.mts-link.ru/api/eventsessions/{event_session_id}/record?withoutCuts=false' return f'https://my.mts-link.ru/api/event-sessions/{event_session_id}/record-files/{recording_id}/flow?withoutCuts=false' @@ -30,7 +33,7 @@ def fetch_json_data(url: str, session_id: Union[str, None]) -> Dict: }, cookies=cookies ) - + try: error_data = response.json() if error_data.get("error", {}).get("code") == 403: @@ -41,7 +44,7 @@ def fetch_json_data(url: str, session_id: Union[str, None]) -> Dict: return except Exception: logging.warning('Server response does not contain JSON.') - + response.raise_for_status() return response.json() @@ -58,7 +61,40 @@ def download_video_chunk(video_url: str, save_directory: str) -> str: total_size = int(response.headers.get('content-length', 0)) with tqdm.tqdm(total=total_size, unit='B', unit_scale=True, desc=f'Downloading {filename}') as progress: - for chunk in response.iter_bytes(chunk_size=8192): + for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE): file.write(chunk) progress.update(len(chunk)) return file_path + + +def download_chunks_parallel( + chunks: List[Tuple[str, float]], + save_directory: str, + max_workers: int = MAX_PARALLEL_DOWNLOADS, +) -> List[Tuple[str, float]]: + """Download multiple chunks in parallel. + + Args: + chunks: List of (url, start_time) tuples. + save_directory: Directory to save files to. + max_workers: Maximum number of parallel downloads. + + Returns: + List of (file_path, start_time) tuples in original order. + """ + results = [None] * len(chunks) + + def _download(index, url, start_time): + path = download_video_chunk(url, save_directory) + return index, path, start_time + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [ + executor.submit(_download, i, url, start_time) + for i, (url, start_time) in enumerate(chunks) + ] + for future in as_completed(futures): + idx, path, start_time = future.result() + results[idx] = (path, start_time) + + return results diff --git a/mtslinker/processor.py b/mtslinker/processor.py index ebd05c6..93308c8 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1,115 +1,308 @@ +import json import logging import os -from typing import Dict, Tuple, List, Union +import shutil +import subprocess +from typing import Dict, List, Tuple, Union -import numpy as np -from moviepy.audio.AudioClip import AudioArrayClip, CompositeAudioClip -from moviepy.audio.io.AudioFileClip import AudioFileClip -from moviepy.video.VideoClip import ColorClip -from moviepy import VideoFileClip, concatenate_videoclips -import warnings -warnings.simplefilter("ignore") +def _check_ffmpeg(): + """Check that ffmpeg and ffprobe are available.""" + for tool in ('ffmpeg', 'ffprobe'): + if not shutil.which(tool): + raise RuntimeError( + f'{tool} is not installed. ' + 'Install it with: apt install ffmpeg / brew install ffmpeg' + ) -from mtslinker.downloader import download_video_chunk +def _ffprobe_streams(file_path: str) -> dict: + """Return ffprobe stream info for a file.""" + result = subprocess.run( + [ + 'ffprobe', '-v', 'quiet', + '-print_format', 'json', + '-show_streams', + '-show_format', + file_path, + ], + capture_output=True, text=True, + ) + if result.returncode != 0: + return {} + return json.loads(result.stdout) + + +def _has_video_stream(file_path: str) -> bool: + """Check if a file contains a video stream.""" + info = _ffprobe_streams(file_path) + for stream in info.get('streams', []): + if stream.get('codec_type') == 'video': + return True + return False + + +def _get_duration(file_path: str) -> float: + """Get duration of a media file in seconds.""" + info = _ffprobe_streams(file_path) + fmt = info.get('format', {}) + if 'duration' in fmt: + return float(fmt['duration']) + for stream in info.get('streams', []): + if 'duration' in stream: + return float(stream['duration']) + return 0.0 + + +def _get_video_params(file_path: str) -> Tuple[int, int, str]: + """Get width, height, and pixel format of the first video stream.""" + info = _ffprobe_streams(file_path) + for stream in info.get('streams', []): + if stream.get('codec_type') == 'video': + w = int(stream.get('width', 1920)) + h = int(stream.get('height', 1080)) + pix_fmt = stream.get('pix_fmt', 'yuv420p') + return w, h, pix_fmt + return 1920, 1080, 'yuv420p' + + +def _generate_black_segment(output_path: str, duration: float, + width: int = 1920, height: int = 1080, + pix_fmt: str = 'yuv420p') -> str: + """Generate a short black video+silent audio segment with ffmpeg.""" + subprocess.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', f'color=c=black:s={width}x{height}:d={duration}:r=25', + '-f', 'lavfi', '-i', f'anullsrc=r=44100:cl=stereo', + '-t', str(duration), + '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'stillimage', + '-pix_fmt', pix_fmt, + '-c:a', 'aac', '-b:a', '128k', + '-shortest', + output_path, + ], + capture_output=True, check=True, + ) + return output_path + + +def _ensure_audio_stream(input_path: str, output_path: str) -> str: + """If the file has no audio stream, add a silent one so concat works.""" + info = _ffprobe_streams(input_path) + has_audio = any( + s.get('codec_type') == 'audio' for s in info.get('streams', []) + ) + if has_audio: + return input_path + + subprocess.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-i', input_path, + '-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo', + '-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k', + '-shortest', + output_path, + ], + capture_output=True, check=True, + ) + return output_path + + +def _normalize_segment(input_path: str, output_path: str, + width: int, height: int, pix_fmt: str) -> str: + """Re-encode a segment to a common format for reliable concatenation.""" + subprocess.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-i', input_path, + '-vf', f'scale={width}:{height}:force_original_aspect_ratio=decrease,' + f'pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black,' + f'setsar=1', + '-pix_fmt', pix_fmt, + '-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '18', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + output_path, + ], + capture_output=True, check=True, + ) + return output_path -def process_video_clips(directory: str, json_data: Dict) -> Tuple[float, List[VideoFileClip], List[AudioFileClip]]: +def process_and_download_clips( + directory: str, json_data: Dict +) -> Tuple[float, List[Tuple[str, float]]]: + """Extract chunk URLs and start times from the API JSON data. + + Returns: + (total_duration, [(url, start_time), ...]) + """ total_duration = float(json_data.get('duration', 0)) if not total_duration: raise ValueError('Duration not found in JSON data.') - video_clips = [] - audio_clips = [] - + chunks = [] for event in json_data.get('eventLogs', []): if isinstance(event, dict): data = event.get('data', {}) if isinstance(data, dict) and 'url' in data: url = data['url'] start_time = event.get('relativeTime', 0) + chunks.append((url, start_time)) - downloaded_file_path = download_video_chunk(url, directory) - try: - video_clip = VideoFileClip(downloaded_file_path, fps_source='fps').with_start(start_time) - video_clips.append(video_clip) - except (KeyError, OSError): - audio_clip = AudioFileClip(downloaded_file_path).with_start(start_time) - audio_clips.append(audio_clip) - logging.info(f'Total duration of clips: {total_duration}') + return total_duration, chunks - return total_duration, video_clips, audio_clips +def compile_final_video( + total_duration: float, + downloaded_files: List[Tuple[str, float]], + directory: str, + output_path: str, + max_duration: Union[int, None], +): + """Concatenate downloaded segments using ffmpeg (no MoviePy re-encoding). -def create_video_with_gaps(total_duration: float, video_clips: List[VideoFileClip]) -> VideoFileClip: - clips = [] - current_time = 0.0 + Steps: + 1. Separate video and audio-only files. + 2. Normalize video segments to a common resolution. + 3. Generate black gap segments where needed. + 4. Concatenate with ffmpeg concat demuxer (-c copy). + 5. Merge audio-only tracks on top. + """ + _check_ffmpeg() + + video_files = [] # (path, start_time) + audio_files = [] # (path, start_time) - for video in video_clips: - if video.start > current_time: - gap_duration = video.start - current_time - if gap_duration > 0: - empty_clip = ColorClip(size=(1920, 1080), color=(0, 0, 0), duration=gap_duration).with_start( - current_time) - clips.append(empty_clip) + for file_path, start_time in downloaded_files: + if _has_video_stream(file_path): + video_files.append((file_path, start_time)) + else: + audio_files.append((file_path, start_time)) - clips.append(video) - current_time = video.end + logging.info(f'Segments: {len(video_files)} video, {len(audio_files)} audio-only') - if current_time < total_duration: - remaining_duration = total_duration - current_time - empty_clip = ColorClip(size=(1920, 1080), color=(0, 0, 0), duration=remaining_duration).with_start(current_time) - clips.append(empty_clip) + if not video_files: + logging.error('No video segments found.') + return - final_video = concatenate_videoclips(clips, method='compose') - logging.info(f'Final video duration: {final_video.duration}') - return final_video + # Determine target resolution from the first video segment + first_video = video_files[0][0] + target_w, target_h, target_pix_fmt = _get_video_params(first_video) + logging.info(f'Target resolution: {target_w}x{target_h}, pix_fmt={target_pix_fmt}') + # Sort by start time + video_files.sort(key=lambda x: x[1]) -def create_audio_with_gaps(total_duration: float, audio_clips: List[AudioFileClip]) -> CompositeAudioClip: - audio_segments = [] - current_time = 0 + # Build the list of segments (normalized videos + gap fillers) + tmp_dir = os.path.join(directory, '_tmp_ffmpeg') + os.makedirs(tmp_dir, exist_ok=True) + + concat_segments = [] + current_time = 0.0 - for audio in audio_clips: - if audio.start > current_time: - gap_duration = audio.start - current_time - if gap_duration > 0: - silence_segment = AudioArrayClip(np.zeros((int(gap_duration * 8000), 2)), fps=8000).with_start( - current_time) - audio_segments.append(silence_segment) + for i, (vpath, start_time) in enumerate(video_files): + # Insert black gap if needed + gap = start_time - current_time + if gap > 0.1: # skip tiny gaps < 100ms + gap_path = os.path.join(tmp_dir, f'gap_{i}.mp4') + _generate_black_segment(gap_path, gap, target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + logging.info(f'Generated {gap:.1f}s black gap before segment {i}') - audio_segments.append(audio) - current_time = audio.end + # Normalize the segment + norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') + _normalize_segment(vpath, norm_path, target_w, target_h, target_pix_fmt) + # Ensure it has an audio stream + with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') + final_seg = _ensure_audio_stream(norm_path, with_audio_path) + concat_segments.append(final_seg) - if current_time < total_duration: - remaining_duration = total_duration - current_time - silence_segment = AudioArrayClip(np.zeros((int(remaining_duration * 8000), 2)), fps=8000).with_start( - current_time) - audio_segments.append(silence_segment) + seg_dur = _get_duration(final_seg) + current_time = start_time + seg_dur - final_audio = CompositeAudioClip(audio_segments) - logging.info(f'Total audio duration: {final_audio.duration}') - return final_audio + # Trailing gap + if current_time < total_duration - 0.1: + gap_path = os.path.join(tmp_dir, 'gap_end.mp4') + _generate_black_segment(gap_path, total_duration - current_time, + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + # Write concat list + concat_list_path = os.path.join(tmp_dir, 'concat.txt') + with open(concat_list_path, 'w') as f: + for seg in concat_segments: + f.write(f"file '{os.path.abspath(seg)}'\n") -def compile_final_video(total_duration: float, video_clips: List[VideoFileClip], audio_clips: List[AudioFileClip], - output_path: str, max_duration: Union[int, None]): - video_result = create_video_with_gaps(total_duration, video_clips) + # Concatenate with ffmpeg concat demuxer (stream copy - no re-encoding) + video_only_path = os.path.join(tmp_dir, 'video_concat.mp4') + logging.info(f'Concatenating {len(concat_segments)} segments...') - if audio_clips: - combined_audio = create_audio_with_gaps(total_duration, audio_clips) - video_result = video_result.with_audio(combined_audio) + concat_cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-f', 'concat', '-safe', '0', + '-i', concat_list_path, + '-c', 'copy', + ] if max_duration: - if video_result.duration > max_duration: - logging.info(f'Duration limit! Crop!') - video_result = video_result.subclip(0, max_duration) - - video_result.write_videofile( - output_path, - codec='libx264', - audio_codec='aac', - preset='ultrafast', - threads=os.cpu_count() + concat_cmd.extend(['-t', str(max_duration)]) + + concat_cmd.append(video_only_path) + subprocess.run(concat_cmd, check=True) + + # If there are audio-only tracks, overlay them + if audio_files: + logging.info(f'Merging {len(audio_files)} audio-only tracks...') + result_path = _merge_audio_tracks(video_only_path, audio_files, tmp_dir, output_path) + else: + # Just move/copy the result + shutil.move(video_only_path, output_path) + result_path = output_path + + # Cleanup temp files + shutil.rmtree(tmp_dir, ignore_errors=True) + logging.info(f'Final video saved to {result_path}') + + +def _merge_audio_tracks( + video_path: str, + audio_files: List[Tuple[str, float]], + tmp_dir: str, + output_path: str, +) -> str: + """Merge audio-only tracks on top of the concatenated video using ffmpeg.""" + # Build a complex filter to mix audio tracks with delays + inputs = ['-i', video_path] + filter_parts = [] + audio_labels = ['[0:a]'] # existing audio from video + + for i, (apath, start_time) in enumerate(audio_files): + inputs.extend(['-i', apath]) + input_idx = i + 1 + delay_ms = int(start_time * 1000) + filter_parts.append( + f'[{input_idx}:a]adelay={delay_ms}|{delay_ms}[a{input_idx}]' + ) + audio_labels.append(f'[a{input_idx}]') + + mix_filter = ';'.join(filter_parts) + if mix_filter: + mix_filter += ';' + mix_filter += ''.join(audio_labels) + f'amix=inputs={len(audio_labels)}:normalize=0[aout]' + + subprocess.run( + [ + 'ffmpeg', '-y', '-v', 'warning', + *inputs, + '-filter_complex', mix_filter, + '-map', '0:v', '-map', '[aout]', + '-c:v', 'copy', + '-c:a', 'aac', '-b:a', '192k', + output_path, + ], + check=True, ) + return output_path diff --git a/mtslinker/webinar.py b/mtslinker/webinar.py index 5bab43c..e5fa10b 100644 --- a/mtslinker/webinar.py +++ b/mtslinker/webinar.py @@ -2,15 +2,19 @@ import os import re -from mtslinker.downloader import construct_json_data_url, fetch_json_data -from mtslinker.processor import compile_final_video, process_video_clips +from mtslinker.downloader import ( + construct_json_data_url, + fetch_json_data, + download_chunks_parallel, +) +from mtslinker.processor import compile_final_video, process_and_download_clips from mtslinker.utils import create_directory_if_not_exists def fetch_webinar_data(event_sessions: str, record_id: str, session_id=None, max_duration=None): json_data_url = construct_json_data_url(event_session_id=event_sessions, recording_id=record_id) json_data = fetch_json_data(url=json_data_url, session_id=session_id) - + if not json_data: logging.error('Failed to fetch webinar data. Check the session ID or URL.') return @@ -19,11 +23,14 @@ def fetch_webinar_data(event_sessions: str, record_id: str, session_id=None, max directory = create_directory_if_not_exists(sanitized_name) output_video_path = os.path.join(directory, f'{sanitized_name}.mp4') - total_duration, video_clips, audio_clips = process_video_clips(directory, json_data) - logging.info( - f'Downloaded and processed {len(video_clips) + len(audio_clips)} files ({total_duration} sec) for merging.') + total_duration, chunks = process_and_download_clips(directory, json_data) + logging.info(f'Found {len(chunks)} chunks to download ({total_duration} sec total)') + + # Download all chunks in parallel + downloaded_files = download_chunks_parallel(chunks, directory) + logging.info(f'Downloaded {len(downloaded_files)} files, starting merge...') - compile_final_video(total_duration, video_clips, audio_clips, output_video_path, max_duration) + compile_final_video(total_duration, downloaded_files, directory, output_video_path, max_duration) logging.info(f'Final video saved to {output_video_path}') - + return 1 diff --git a/requirements.txt b/requirements.txt index fd6c954..cd69fab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ tqdm==4.66.6 httpx==0.27.2 -moviepy==2.1.1 diff --git a/setup.py b/setup.py index dfb3283..d70641b 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,10 @@ setup( name='mtslinker', - version='1.0.0', + version='2.0.0', packages=find_packages(), install_requires=[ 'httpx>=0.27.2', - 'moviepy>=1.0.3', 'tqdm>=4.66.6', ], entry_points={ From cdcb119384eb8af3e41b654954a10a5007de9ab9 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 25 Mar 2026 16:19:19 +0000 Subject: [PATCH 02/65] Download video via HLS when storage mp4 is audio-only Some MTS Link recordings store only audio in the direct mp4 files, while the HLS delivery endpoint has both video and audio streams. Check the HLS playlist for a video track and download via ffmpeg when detected. --- mtslinker/downloader.py | 42 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/mtslinker/downloader.py b/mtslinker/downloader.py index 412e001..def4f9c 100644 --- a/mtslinker/downloader.py +++ b/mtslinker/downloader.py @@ -1,4 +1,5 @@ import os +import subprocess from typing import Dict, List, Tuple, Union from concurrent.futures import ThreadPoolExecutor, as_completed @@ -11,6 +12,41 @@ MAX_PARALLEL_DOWNLOADS = 4 +def _storage_to_hls_url(storage_url: str) -> str: + """Convert events-storage URL to events-delivery-records HLS URL.""" + return ( + storage_url + .replace('events-storage.webinar.ru/api-storage/files/wowza/', + 'events-delivery-records.webinar.ru/record/') + + '/playlist.m3u8' + ) + + +def _hls_has_video(hls_url: str) -> bool: + """Check if an HLS playlist has a video track.""" + try: + with httpx.Client(timeout=httpx.Timeout(10)) as client: + r = client.get(hls_url, headers={'User-Agent': 'Mozilla/5.0'}) + return 'v1/' in r.text + except Exception: + return False + + +def download_hls_chunk(hls_url: str, save_path: str) -> str: + """Download an HLS stream to mp4 using ffmpeg.""" + if not os.path.exists(save_path): + subprocess.run( + [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', hls_url, + '-c', 'copy', + save_path, + ], + check=True, + ) + return save_path + + def construct_json_data_url(event_session_id: str, recording_id: str) -> str: if not event_session_id: raise ValueError('Missing webinar event session ID.') @@ -54,6 +90,12 @@ def download_video_chunk(video_url: str, save_directory: str) -> str: file_path = os.path.join(save_directory, filename) if not os.path.exists(file_path): + # Check if HLS version has video (storage mp4 may be audio-only) + hls_url = _storage_to_hls_url(video_url) + if _hls_has_video(hls_url): + logging.info(f'HLS has video for {filename}, downloading via ffmpeg') + return download_hls_chunk(hls_url, file_path) + with open(file_path, 'wb') as file: with httpx.Client(timeout=TIMEOUT_SETTINGS) as client: with client.stream('GET', video_url) as response: From 18d46b9475fe84820c41c09cfd8029a6b6d7f2fc Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 25 Mar 2026 20:23:47 +0000 Subject: [PATCH 03/65] Fix OOM crash during audio merge by batching ffmpeg amix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old _merge_audio_tracks passed all audio files (up to 63+) as simultaneous inputs to a single ffmpeg amix command, which required ffmpeg to hold all delayed audio streams in memory for the full recording duration — causing OOM kills on long recordings. Now audio tracks are pre-delayed individually, then mixed via tree reduction in batches of 8. Also routes all subprocess calls through _run_ffmpeg which logs stderr on failure instead of silently swallowing it. --- mtslinker/processor.py | 160 +++++++++++++++++++++++++++++++++-------- 1 file changed, 132 insertions(+), 28 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 93308c8..17a0ef8 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -5,6 +5,8 @@ import subprocess from typing import Dict, List, Tuple, Union +AUDIO_MERGE_BATCH_SIZE = 8 + def _check_ffmpeg(): """Check that ffmpeg and ffprobe are available.""" @@ -66,11 +68,26 @@ def _get_video_params(file_path: str) -> Tuple[int, int, str]: return 1920, 1080, 'yuv420p' +def _run_ffmpeg(cmd: list, description: str = 'ffmpeg'): + """Run an ffmpeg command, logging stderr on failure.""" + logging.debug(f'Running {description}: {" ".join(cmd[:6])}...') + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + stderr = result.stderr.strip() if result.stderr else '(no stderr)' + logging.error(f'{description} failed (exit {result.returncode}):\n{stderr}') + raise subprocess.CalledProcessError( + result.returncode, cmd[0], result.stdout, result.stderr + ) + if result.stderr and result.stderr.strip(): + logging.debug(f'{description} stderr: {result.stderr.strip()[:500]}') + return result + + def _generate_black_segment(output_path: str, duration: float, width: int = 1920, height: int = 1080, pix_fmt: str = 'yuv420p') -> str: """Generate a short black video+silent audio segment with ffmpeg.""" - subprocess.run( + _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', '-f', 'lavfi', '-i', f'color=c=black:s={width}x{height}:d={duration}:r=25', @@ -82,7 +99,7 @@ def _generate_black_segment(output_path: str, duration: float, '-shortest', output_path, ], - capture_output=True, check=True, + description=f'generate black segment ({duration:.1f}s)', ) return output_path @@ -96,7 +113,7 @@ def _ensure_audio_stream(input_path: str, output_path: str) -> str: if has_audio: return input_path - subprocess.run( + _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', '-i', input_path, @@ -105,7 +122,7 @@ def _ensure_audio_stream(input_path: str, output_path: str) -> str: '-shortest', output_path, ], - capture_output=True, check=True, + description='add silent audio stream', ) return output_path @@ -113,7 +130,7 @@ def _ensure_audio_stream(input_path: str, output_path: str) -> str: def _normalize_segment(input_path: str, output_path: str, width: int, height: int, pix_fmt: str) -> str: """Re-encode a segment to a common format for reliable concatenation.""" - subprocess.run( + _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', '-i', input_path, @@ -126,7 +143,7 @@ def _normalize_segment(input_path: str, output_path: str, '-r', '25', output_path, ], - capture_output=True, check=True, + description=f'normalize segment {os.path.basename(input_path)}', ) return output_path @@ -169,7 +186,7 @@ def compile_final_video( 2. Normalize video segments to a common resolution. 3. Generate black gap segments where needed. 4. Concatenate with ffmpeg concat demuxer (-c copy). - 5. Merge audio-only tracks on top. + 5. Merge audio-only tracks on top (in batches to avoid OOM). """ _check_ffmpeg() @@ -251,12 +268,14 @@ def compile_final_video( concat_cmd.extend(['-t', str(max_duration)]) concat_cmd.append(video_only_path) - subprocess.run(concat_cmd, check=True) + _run_ffmpeg(concat_cmd, description=f'concat {len(concat_segments)} segments') # If there are audio-only tracks, overlay them if audio_files: logging.info(f'Merging {len(audio_files)} audio-only tracks...') - result_path = _merge_audio_tracks(video_only_path, audio_files, tmp_dir, output_path) + result_path = _merge_audio_tracks( + video_only_path, audio_files, tmp_dir, output_path + ) else: # Just move/copy the result shutil.move(video_only_path, output_path) @@ -273,36 +292,121 @@ def _merge_audio_tracks( tmp_dir: str, output_path: str, ) -> str: - """Merge audio-only tracks on top of the concatenated video using ffmpeg.""" - # Build a complex filter to mix audio tracks with delays - inputs = ['-i', video_path] - filter_parts = [] - audio_labels = ['[0:a]'] # existing audio from video - + """Merge audio-only tracks on top of the concatenated video. + + To avoid OOM from passing dozens of inputs to a single ffmpeg amix, + we pre-mix all audio tracks into one WAV in batches, then overlay + that single track onto the video. + + Strategy: + 1. Each audio file is individually converted to a delayed WAV + (silence-padded to its start_time offset). + 2. WAVs are mixed in batches of AUDIO_MERGE_BATCH_SIZE using amix. + 3. Batch results are mixed together (tree reduction) until one + remains. + 4. The final mixed audio is overlaid onto the video. + """ + video_duration = _get_duration(video_path) + batch_size = AUDIO_MERGE_BATCH_SIZE + + # Step 1: Convert each audio track to a delayed mono/stereo WAV. + # We use adelay + apad + atrim so each file is positioned at its + # correct offset and truncated to video duration. This way amix + # inputs are all the same length and ffmpeg doesn't need to buffer + # indefinitely. + delayed_paths = [] for i, (apath, start_time) in enumerate(audio_files): - inputs.extend(['-i', apath]) - input_idx = i + 1 + delayed_path = os.path.join(tmp_dir, f'audio_delayed_{i}.wav') delay_ms = int(start_time * 1000) - filter_parts.append( - f'[{input_idx}:a]adelay={delay_ms}|{delay_ms}[a{input_idx}]' + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-i', apath, + '-af', ( + f'adelay={delay_ms}|{delay_ms},' + f'apad=whole_dur={video_duration},' + f'atrim=0:{video_duration},' + f'asetpts=PTS-STARTPTS' + ), + '-ar', '44100', '-ac', '2', + delayed_path, + ], + description=f'delay audio track {i}/{len(audio_files)} ' + f'(offset {start_time:.1f}s)', + ) + delayed_paths.append(delayed_path) + logging.debug(f'Prepared delayed audio {i+1}/{len(audio_files)}') + + # Step 2+3: Tree-reduce via amix in batches. + round_num = 0 + current_paths = delayed_paths + while len(current_paths) > 1: + round_num += 1 + next_paths = [] + for batch_start in range(0, len(current_paths), batch_size): + batch = current_paths[batch_start:batch_start + batch_size] + if len(batch) == 1: + next_paths.append(batch[0]) + continue + + batch_out = os.path.join( + tmp_dir, f'audio_mix_r{round_num}_b{batch_start}.wav' + ) + inputs = [] + for bp in batch: + inputs.extend(['-i', bp]) + + labels = ''.join(f'[{j}:a]' for j in range(len(batch))) + amix_filter = ( + f'{labels}amix=inputs={len(batch)}' + f':duration=longest:normalize=0' + ) + + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-filter_complex', amix_filter, + '-ar', '44100', '-ac', '2', + batch_out, + ], + description=f'amix round {round_num}, batch {batch_start} ' + f'({len(batch)} tracks)', + ) + next_paths.append(batch_out) + + # Clean up consumed intermediate files (not the originals + # from round 0 — those are the delayed WAVs we still need + # if something goes wrong, but they're in tmp_dir anyway). + if round_num > 1: + for bp in batch: + try: + os.remove(bp) + except OSError: + pass + + logging.info( + f'Audio mix round {round_num}: {len(current_paths)} -> ' + f'{len(next_paths)} tracks' ) - audio_labels.append(f'[a{input_idx}]') + current_paths = next_paths - mix_filter = ';'.join(filter_parts) - if mix_filter: - mix_filter += ';' - mix_filter += ''.join(audio_labels) + f'amix=inputs={len(audio_labels)}:normalize=0[aout]' + mixed_audio_path = current_paths[0] - subprocess.run( + # Step 4: Overlay the single mixed audio track onto the video. + logging.info('Overlaying mixed audio onto video...') + _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'warning', - *inputs, - '-filter_complex', mix_filter, + '-i', video_path, + '-i', mixed_audio_path, + '-filter_complex', + '[0:a][1:a]amix=inputs=2:duration=first:normalize=0[aout]', '-map', '0:v', '-map', '[aout]', '-c:v', 'copy', '-c:a', 'aac', '-b:a', '192k', output_path, ], - check=True, + description='overlay mixed audio onto video', ) return output_path From b3c6a7b1e2586c988bf0ed9281e9ba4f011cfa87 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 25 Mar 2026 21:08:38 +0000 Subject: [PATCH 04/65] Fix 3x duration inflation from overlapping parallel video streams Recordings with multiple simultaneous feeds (webcam + screen share) have segments with overlapping timestamps. The old code laid them out sequentially, turning a 3hr recording into 10+ hours of concat video. This also caused the audio merge WAVs to be padded to 10hrs each, requiring ~400GB of disk. Added _deduplicate_overlapping() which keeps only the longest segment per time window (186 -> 7 segments in a real test case). Also pass total_duration from the API to _merge_audio_tracks so WAVs are padded to the correct recording length, not the (potentially inflated) concat file duration. --- mtslinker/processor.py | 72 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 17a0ef8..406306a 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -172,6 +172,63 @@ def process_and_download_clips( return total_duration, chunks +def _deduplicate_overlapping( + video_files: List[Tuple[str, float]], +) -> List[Tuple[str, float]]: + """Remove overlapping video segments, keeping the longest per time window. + + Recordings often have parallel streams (webcam + screen share) at the + same timestamp. Laying them out sequentially inflates the duration. + This keeps only one segment per overlapping group — the longest one — + so the final timeline matches real elapsed time. + """ + if not video_files: + return video_files + + # Annotate with duration + annotated = [] # (path, start, duration, end) + for path, start in video_files: + dur = _get_duration(path) + annotated.append((path, start, dur, start + dur)) + + # Sort by start time, then longest first so we prefer longer segments + annotated.sort(key=lambda x: (x[1], -x[2])) + + kept = [] # (path, start, duration, end) + for seg in annotated: + path, start, dur, end = seg + if not kept: + kept.append(seg) + continue + + _, prev_start, _, prev_end = kept[-1] + + if start >= prev_end - 0.5: + # No meaningful overlap — keep this segment + kept.append(seg) + else: + # Overlaps with the previous winner — keep whichever is longer + if dur > kept[-1][2]: + logging.debug( + f'Dedup: replacing {kept[-1][2]:.1f}s segment at ' + f'{prev_start:.1f}s with {dur:.1f}s segment at {start:.1f}s' + ) + kept[-1] = seg + else: + logging.debug( + f'Dedup: skipping {dur:.1f}s segment at {start:.1f}s ' + f'(covered by {kept[-1][2]:.1f}s segment at {prev_start:.1f}s)' + ) + + original = len(video_files) + deduped = len(kept) + if original != deduped: + logging.info(f'Dedup: {original} -> {deduped} segments ' + f'(removed {original - deduped} overlapping)') + + return [(path, start) for path, start, dur, end in kept] + + def compile_final_video( total_duration: float, downloaded_files: List[Tuple[str, float]], @@ -213,6 +270,15 @@ def compile_final_video( # Sort by start time video_files.sort(key=lambda x: x[1]) + # Deduplicate overlapping segments: recordings often have parallel + # streams (webcam + screen share) running at the same time. If we + # concatenate them all sequentially the output is 3-4x too long. + # Strategy: walk through sorted segments; when a new segment starts + # before the current winner ends, keep whichever is longer and + # discard the shorter one. + video_files = _deduplicate_overlapping(video_files) + logging.info(f'After dedup: {len(video_files)} non-overlapping video segments') + # Build the list of segments (normalized videos + gap fillers) tmp_dir = os.path.join(directory, '_tmp_ffmpeg') os.makedirs(tmp_dir, exist_ok=True) @@ -274,7 +340,8 @@ def compile_final_video( if audio_files: logging.info(f'Merging {len(audio_files)} audio-only tracks...') result_path = _merge_audio_tracks( - video_only_path, audio_files, tmp_dir, output_path + video_only_path, audio_files, tmp_dir, output_path, + total_duration, ) else: # Just move/copy the result @@ -291,6 +358,7 @@ def _merge_audio_tracks( audio_files: List[Tuple[str, float]], tmp_dir: str, output_path: str, + total_duration: float = 0, ) -> str: """Merge audio-only tracks on top of the concatenated video. @@ -306,7 +374,7 @@ def _merge_audio_tracks( remains. 4. The final mixed audio is overlaid onto the video. """ - video_duration = _get_duration(video_path) + video_duration = total_duration or _get_duration(video_path) batch_size = AUDIO_MERGE_BATCH_SIZE # Step 1: Convert each audio track to a delayed mono/stereo WAV. From 12db22d1b973fe0c6f4552254cac2bfbc2d02a5d Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 25 Mar 2026 21:50:07 +0000 Subject: [PATCH 05/65] Fix disk exhaustion: use compressed m4a instead of WAVs for audio merge The previous approach materialized each audio track as a full-duration WAV (~1.8GB each for a 3hr recording). With 63 tracks that's ~113GB, filling the disk and crashing with "No space left on device". Now audio tracks are mixed in batches directly with adelay inside the ffmpeg filter graph, outputting compressed m4a (~15MB each). No intermediate WAVs are created. Batch results are tree-reduced and intermediates are deleted immediately after each round. --- mtslinker/processor.py | 115 ++++++++++++++++++++++++----------------- 1 file changed, 69 insertions(+), 46 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 406306a..eef8961 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -362,52 +362,77 @@ def _merge_audio_tracks( ) -> str: """Merge audio-only tracks on top of the concatenated video. - To avoid OOM from passing dozens of inputs to a single ffmpeg amix, - we pre-mix all audio tracks into one WAV in batches, then overlay - that single track onto the video. + To avoid OOM (too many ffmpeg inputs) and disk exhaustion (huge WAVs), + we mix audio in small batches directly with adelay inside the filter, + outputting compressed m4a. Each batch produces one small file, and + consumed intermediates are deleted immediately. Strategy: - 1. Each audio file is individually converted to a delayed WAV - (silence-padded to its start_time offset). - 2. WAVs are mixed in batches of AUDIO_MERGE_BATCH_SIZE using amix. - 3. Batch results are mixed together (tree reduction) until one - remains. - 4. The final mixed audio is overlaid onto the video. + 1. Mix audio files in batches of AUDIO_MERGE_BATCH_SIZE, applying + adelay inside the filter graph (no pre-materialized delayed files). + 2. Tree-reduce batch outputs until one mixed track remains. + 3. Overlay the single mixed audio onto the video. """ video_duration = total_duration or _get_duration(video_path) batch_size = AUDIO_MERGE_BATCH_SIZE - # Step 1: Convert each audio track to a delayed mono/stereo WAV. - # We use adelay + apad + atrim so each file is positioned at its - # correct offset and truncated to video duration. This way amix - # inputs are all the same length and ffmpeg doesn't need to buffer - # indefinitely. - delayed_paths = [] - for i, (apath, start_time) in enumerate(audio_files): - delayed_path = os.path.join(tmp_dir, f'audio_delayed_{i}.wav') - delay_ms = int(start_time * 1000) + # Step 1: Mix in batches with inline adelay (no intermediate WAVs). + # Each batch takes up to batch_size original audio files, applies + # adelay per track, mixes them, and writes a compressed m4a. + batch_outputs = [] # (path, effective_start=0 since delay is baked in) + for batch_idx, batch_start in enumerate( + range(0, len(audio_files), batch_size) + ): + batch = audio_files[batch_start:batch_start + batch_size] + batch_out = os.path.join(tmp_dir, f'audio_batch_{batch_idx}.m4a') + + inputs = [] + filter_parts = [] + mix_labels = [] + for j, (apath, start_time) in enumerate(batch): + inputs.extend(['-i', apath]) + delay_ms = int(start_time * 1000) + label = f'a{j}' + filter_parts.append( + f'[{j}:a]adelay={delay_ms}|{delay_ms},' + f'apad=whole_dur={video_duration},' + f'atrim=0:{video_duration},' + f'asetpts=PTS-STARTPTS[{label}]' + ) + mix_labels.append(f'[{label}]') + + if len(batch) == 1: + # Single track — just delay and encode, no amix needed + filter_graph = filter_parts[0].rsplit('[', 1)[0] # strip label + else: + filter_graph = ( + ';'.join(filter_parts) + ';' + + ''.join(mix_labels) + + f'amix=inputs={len(batch)}:duration=longest:normalize=0' + ) + _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', - '-i', apath, - '-af', ( - f'adelay={delay_ms}|{delay_ms},' - f'apad=whole_dur={video_duration},' - f'atrim=0:{video_duration},' - f'asetpts=PTS-STARTPTS' - ), + *inputs, + '-filter_complex', filter_graph, + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - delayed_path, + batch_out, ], - description=f'delay audio track {i}/{len(audio_files)} ' - f'(offset {start_time:.1f}s)', + description=f'amix batch {batch_idx+1} ' + f'({len(batch)} tracks, offset {batch[0][1]:.0f}-' + f'{batch[-1][1]:.0f}s)', + ) + batch_outputs.append(batch_out) + logging.info( + f'Audio batch {batch_idx+1}/' + f'{(len(audio_files) + batch_size - 1) // batch_size} done' ) - delayed_paths.append(delayed_path) - logging.debug(f'Prepared delayed audio {i+1}/{len(audio_files)}') - # Step 2+3: Tree-reduce via amix in batches. + # Step 2: Tree-reduce batch outputs until one remains. round_num = 0 - current_paths = delayed_paths + current_paths = batch_outputs while len(current_paths) > 1: round_num += 1 next_paths = [] @@ -418,7 +443,7 @@ def _merge_audio_tracks( continue batch_out = os.path.join( - tmp_dir, f'audio_mix_r{round_num}_b{batch_start}.wav' + tmp_dir, f'audio_reduce_r{round_num}_b{batch_start}.m4a' ) inputs = [] for bp in batch: @@ -435,33 +460,31 @@ def _merge_audio_tracks( 'ffmpeg', '-y', '-v', 'error', *inputs, '-filter_complex', amix_filter, + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', batch_out, ], - description=f'amix round {round_num}, batch {batch_start} ' - f'({len(batch)} tracks)', + description=f'amix reduce round {round_num}, ' + f'{len(batch)} tracks', ) next_paths.append(batch_out) - # Clean up consumed intermediate files (not the originals - # from round 0 — those are the delayed WAVs we still need - # if something goes wrong, but they're in tmp_dir anyway). - if round_num > 1: - for bp in batch: - try: - os.remove(bp) - except OSError: - pass + # Delete consumed intermediates + for bp in batch: + try: + os.remove(bp) + except OSError: + pass logging.info( - f'Audio mix round {round_num}: {len(current_paths)} -> ' + f'Audio reduce round {round_num}: {len(current_paths)} -> ' f'{len(next_paths)} tracks' ) current_paths = next_paths mixed_audio_path = current_paths[0] - # Step 4: Overlay the single mixed audio track onto the video. + # Step 3: Overlay the single mixed audio track onto the video. logging.info('Overlaying mixed audio onto video...') _run_ffmpeg( [ From 03db61edd2fc327c34eca13d626fc450b3dba69d Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 26 Mar 2026 07:09:25 +0000 Subject: [PATCH 06/65] Handle corrupt downloads: validate, retry, and skip bad files - Add _validate_downloaded_file() to check files with ffprobe after download - Re-download corrupt files (missing moov atom) up to 2 retries - Validate existing cached files on disk, re-download if corrupt - Add _is_valid_media() in processor to skip corrupt files during classification - Audio batch mixing catches errors and skips failed batches instead of crashing - If all audio batches fail, output video without audio overlay --- mtslinker/downloader.py | 54 +++++++++++++++++++++++++++---------- mtslinker/processor.py | 59 +++++++++++++++++++++++++++++++---------- 2 files changed, 85 insertions(+), 28 deletions(-) diff --git a/mtslinker/downloader.py b/mtslinker/downloader.py index def4f9c..7dffb6c 100644 --- a/mtslinker/downloader.py +++ b/mtslinker/downloader.py @@ -85,27 +85,53 @@ def fetch_json_data(url: str, session_id: Union[str, None]) -> Dict: return response.json() -def download_video_chunk(video_url: str, save_directory: str) -> str: +def _validate_downloaded_file(file_path: str) -> bool: + """Quick check that a downloaded media file is not corrupt.""" + result = subprocess.run( + ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', + '-of', 'csv=p=0', file_path], + capture_output=True, text=True, + ) + return result.returncode == 0 + + +def download_video_chunk(video_url: str, save_directory: str, max_retries: int = 2) -> str: filename = os.path.basename(video_url) file_path = os.path.join(save_directory, filename) - if not os.path.exists(file_path): + if os.path.exists(file_path): + if _validate_downloaded_file(file_path): + return file_path + logging.warning(f'Existing file corrupt, re-downloading: {filename}') + os.remove(file_path) + + for attempt in range(max_retries + 1): # Check if HLS version has video (storage mp4 may be audio-only) hls_url = _storage_to_hls_url(video_url) if _hls_has_video(hls_url): logging.info(f'HLS has video for {filename}, downloading via ffmpeg') - return download_hls_chunk(hls_url, file_path) - - with open(file_path, 'wb') as file: - with httpx.Client(timeout=TIMEOUT_SETTINGS) as client: - with client.stream('GET', video_url) as response: - response.raise_for_status() - total_size = int(response.headers.get('content-length', 0)) - with tqdm.tqdm(total=total_size, unit='B', unit_scale=True, - desc=f'Downloading {filename}') as progress: - for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE): - file.write(chunk) - progress.update(len(chunk)) + download_hls_chunk(hls_url, file_path) + else: + with open(file_path, 'wb') as file: + with httpx.Client(timeout=TIMEOUT_SETTINGS) as client: + with client.stream('GET', video_url) as response: + response.raise_for_status() + total_size = int(response.headers.get('content-length', 0)) + with tqdm.tqdm(total=total_size, unit='B', unit_scale=True, + desc=f'Downloading {filename}') as progress: + for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE): + file.write(chunk) + progress.update(len(chunk)) + + if _validate_downloaded_file(file_path): + return file_path + + if attempt < max_retries: + logging.warning(f'Downloaded file corrupt (attempt {attempt+1}), retrying: {filename}') + os.remove(file_path) + else: + logging.error(f'File still corrupt after {max_retries+1} attempts: {filename}') + return file_path diff --git a/mtslinker/processor.py b/mtslinker/processor.py index eef8961..fbbfefc 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -18,6 +18,19 @@ def _check_ffmpeg(): ) +def _is_valid_media(file_path: str) -> bool: + """Check if a media file is valid (not corrupt / has moov atom).""" + result = subprocess.run( + ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', + '-of', 'csv=p=0', file_path], + capture_output=True, text=True, + ) + if result.returncode != 0: + logging.warning(f'Corrupt/invalid file: {file_path}: {result.stderr.strip()[:200]}') + return False + return True + + def _ffprobe_streams(file_path: str) -> dict: """Return ffprobe stream info for a file.""" result = subprocess.run( @@ -249,13 +262,19 @@ def compile_final_video( video_files = [] # (path, start_time) audio_files = [] # (path, start_time) + skipped = 0 for file_path, start_time in downloaded_files: + if not _is_valid_media(file_path): + skipped += 1 + continue if _has_video_stream(file_path): video_files.append((file_path, start_time)) else: audio_files.append((file_path, start_time)) + if skipped: + logging.warning(f'Skipped {skipped} corrupt/invalid files') logging.info(f'Segments: {len(video_files)} video, {len(audio_files)} audio-only') if not video_files: @@ -411,25 +430,37 @@ def _merge_audio_tracks( + f'amix=inputs={len(batch)}:duration=longest:normalize=0' ) - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - *inputs, - '-filter_complex', filter_graph, - '-c:a', 'aac', '-b:a', '128k', - '-ar', '44100', '-ac', '2', - batch_out, - ], - description=f'amix batch {batch_idx+1} ' - f'({len(batch)} tracks, offset {batch[0][1]:.0f}-' - f'{batch[-1][1]:.0f}s)', - ) - batch_outputs.append(batch_out) + try: + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-filter_complex', filter_graph, + '-c:a', 'aac', '-b:a', '128k', + '-ar', '44100', '-ac', '2', + batch_out, + ], + description=f'amix batch {batch_idx+1} ' + f'({len(batch)} tracks, offset {batch[0][1]:.0f}-' + f'{batch[-1][1]:.0f}s)', + ) + batch_outputs.append(batch_out) + except subprocess.CalledProcessError: + logging.warning( + f'Audio batch {batch_idx+1} failed, skipping ' + f'{len(batch)} tracks' + ) logging.info( f'Audio batch {batch_idx+1}/' f'{(len(audio_files) + batch_size - 1) // batch_size} done' ) + # If no batches succeeded, skip audio overlay entirely. + if not batch_outputs: + logging.warning('All audio batches failed, skipping audio overlay') + shutil.move(video_path, output_path) + return output_path + # Step 2: Tree-reduce batch outputs until one remains. round_num = 0 current_paths = batch_outputs From 49620da821c0412fca27f50b00d15a56a1b778fe Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 26 Mar 2026 21:06:32 +0000 Subject: [PATCH 07/65] Add presentation slide overlay to video output Extract presentation.update events from the MTS API to get slide images and their timestamps. Download pre-rendered slide JPGs and composite them with the webcam video in a 1280x720 layout: - Left 960px: presentation slide - Top-right 320x180: webcam - Slides are pre-encoded as 1fps video segments and concatenated into a single track, then overlaid with the webcam in one pass. Recordings without presentations are unaffected (existing behavior). --- mtslinker/downloader.py | 39 +++++++++ mtslinker/processor.py | 174 +++++++++++++++++++++++++++++++++++++--- mtslinker/webinar.py | 13 ++- 3 files changed, 214 insertions(+), 12 deletions(-) diff --git a/mtslinker/downloader.py b/mtslinker/downloader.py index 7dffb6c..2c91859 100644 --- a/mtslinker/downloader.py +++ b/mtslinker/downloader.py @@ -135,6 +135,45 @@ def download_video_chunk(video_url: str, save_directory: str, max_retries: int = return file_path +def download_slide_images( + slide_events: List[Dict], + save_directory: str, +) -> List[Dict]: + """Download slide images and add local_path to each event. + + Deduplicates by URL so each unique slide is downloaded once. + """ + slides_dir = os.path.join(save_directory, 'slides') + os.makedirs(slides_dir, exist_ok=True) + + url_to_path = {} + for se in slide_events: + url = se['slide_url'] + if url in url_to_path: + continue + local_path = os.path.join(slides_dir, f"slide_{se['slide_number']}.jpg") + if not os.path.exists(local_path): + try: + with httpx.Client(timeout=httpx.Timeout(30)) as client: + r = client.get(url) + r.raise_for_status() + with open(local_path, 'wb') as f: + f.write(r.content) + except Exception as e: + logging.warning(f"Failed to download slide {se['slide_number']}: {e}") + continue + url_to_path[url] = local_path + + result = [] + for se in slide_events: + path = url_to_path.get(se['slide_url']) + if path: + result.append({**se, 'local_path': path}) + + logging.info(f'Downloaded {len(url_to_path)} unique slide images') + return result + + def download_chunks_parallel( chunks: List[Tuple[str, float]], save_directory: str, diff --git a/mtslinker/processor.py b/mtslinker/processor.py index fbbfefc..75a2401 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -163,26 +163,57 @@ def _normalize_segment(input_path: str, output_path: str, def process_and_download_clips( directory: str, json_data: Dict -) -> Tuple[float, List[Tuple[str, float]]]: - """Extract chunk URLs and start times from the API JSON data. +) -> Tuple[float, List[Tuple[str, float]], List[Dict]]: + """Extract chunk URLs, start times, and presentation slides from the API JSON. Returns: - (total_duration, [(url, start_time), ...]) + (total_duration, [(url, start_time), ...], [slide_event, ...]) + + Each slide_event is: + {"time": float, "slide_number": int, "slide_url": str} """ total_duration = float(json_data.get('duration', 0)) if not total_duration: raise ValueError('Duration not found in JSON data.') chunks = [] + slide_events = [] for event in json_data.get('eventLogs', []): - if isinstance(event, dict): - data = event.get('data', {}) - if isinstance(data, dict) and 'url' in data: - url = data['url'] - start_time = event.get('relativeTime', 0) - chunks.append((url, start_time)) + if not isinstance(event, dict): + continue + data = event.get('data', {}) + if not isinstance(data, dict): + continue + + # Media chunks (video/audio) + if 'url' in data: + chunks.append((data['url'], event.get('relativeTime', 0))) + + # Presentation slide changes + if event.get('module') == 'presentation.update': + fr = data.get('fileReference', {}) + if not isinstance(fr, dict): + continue + slide = fr.get('slide', {}) + if not isinstance(slide, dict) or not slide.get('url'): + continue + slide_url = slide['url'] + slide_events.append({ + 'time': event.get('relativeTime', 0), + 'slide_number': slide.get('number', 0), + 'slide_url': slide_url, + }) + + # Deduplicate consecutive identical slides + deduped_slides = [] + for se in slide_events: + if not deduped_slides or se['slide_url'] != deduped_slides[-1]['slide_url']: + deduped_slides.append(se) - return total_duration, chunks + if deduped_slides: + logging.info(f'Found {len(deduped_slides)} presentation slide changes') + + return total_duration, chunks, deduped_slides def _deduplicate_overlapping( @@ -242,12 +273,126 @@ def _deduplicate_overlapping( return [(path, start) for path, start, dur, end in kept] +def _composite_slides( + video_path: str, + slide_events: List[Dict], + output_path: str, + tmp_dir: str, + total_duration: float, +) -> str: + """Composite presentation slides with webcam video. + + Layout (1280x720): + - Left 960px: presentation slide + - Right 320px, top: webcam (320x180) + - Right 320px, below webcam: black + """ + CANVAS_W, CANVAS_H = 1280, 720 + SLIDE_W, SLIDE_H = 960, 720 + CAM_W, CAM_H = 320, 180 + + logging.info(f'Compositing {len(slide_events)} slide changes onto video...') + + # Step 1: Create a slide video track using concat demuxer. + # Each slide becomes a segment of the right duration. + slides_dir = os.path.join(tmp_dir, 'slide_segments') + os.makedirs(slides_dir, exist_ok=True) + + slide_segments = [] + for i, se in enumerate(slide_events): + t_start = se['time'] + t_end = slide_events[i + 1]['time'] if i + 1 < len(slide_events) else total_duration + duration = t_end - t_start + if duration <= 0: + continue + + seg_path = os.path.join(slides_dir, f'seg_{i}.mp4') + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-loop', '1', '-framerate', '1', '-t', str(duration), + '-i', se['local_path'], + '-vf', f'scale={SLIDE_W}:{SLIDE_H}:force_original_aspect_ratio=decrease,' + f'pad={SLIDE_W}:{SLIDE_H}:(ow-iw)/2:(oh-ih)/2:white', + '-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23', + '-pix_fmt', 'yuv420p', '-r', '1', + seg_path, + ], + description=f'slide segment {i+1}/{len(slide_events)}', + ) + slide_segments.append(seg_path) + + # Add black leader if first slide starts after 0 + first_time = slide_events[0]['time'] if slide_events else 0 + if first_time > 0.5: + leader_path = os.path.join(slides_dir, 'leader.mp4') + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', + f'color=c=black:s={SLIDE_W}x{SLIDE_H}:d={first_time}:r=1', + '-c:v', 'libx264', '-preset', 'ultrafast', + '-pix_fmt', 'yuv420p', + leader_path, + ], + description='slide leader (black)', + ) + slide_segments.insert(0, leader_path) + + # Concatenate slide segments into one track + slide_track_path = os.path.join(tmp_dir, 'slide_track.mp4') + concat_list = os.path.join(slides_dir, 'concat.txt') + with open(concat_list, 'w') as f: + for seg in slide_segments: + f.write(f"file '{os.path.abspath(seg)}'\n") + + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-f', 'concat', '-safe', '0', '-i', concat_list, + '-c', 'copy', slide_track_path, + ], + description='concat slide track', + ) + logging.info('Slide track created') + + # Step 2: Combine slide track + webcam into final layout. + # Simple 2-input overlay: slide on left, webcam scaled to top-right. + filter_graph = ( + f'[1:v]scale={CAM_W}:{CAM_H}:force_original_aspect_ratio=decrease,' + f'pad={CAM_W}:{CAM_H}:(ow-iw)/2:(oh-ih)/2:black[webcam];' + f'[0:v]pad={CANVAS_W}:{CANVAS_H}:0:0:black[padded];' + f'[padded][webcam]overlay={SLIDE_W}:0[out]' + ) + + cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', slide_track_path, + '-i', video_path, + '-filter_complex', filter_graph, + '-map', '[out]', '-map', '1:a?', + '-c:v', 'libx264', '-preset', 'fast', '-crf', '23', + '-c:a', 'copy', + '-r', '25', + '-shortest', + output_path, + ] + + _run_ffmpeg(cmd, description='composite slides + webcam') + logging.info('Slide compositing complete') + + # Cleanup slide segments + shutil.rmtree(slides_dir, ignore_errors=True) + return output_path + + def compile_final_video( total_duration: float, downloaded_files: List[Tuple[str, float]], directory: str, output_path: str, max_duration: Union[int, None], + slide_events: List[Dict] = None, ): """Concatenate downloaded segments using ffmpeg (no MoviePy re-encoding). @@ -355,6 +500,15 @@ def compile_final_video( concat_cmd.append(video_only_path) _run_ffmpeg(concat_cmd, description=f'concat {len(concat_segments)} segments') + # Composite slides if presentation data exists + if slide_events: + composited_path = os.path.join(tmp_dir, 'video_composited.mp4') + _composite_slides( + video_only_path, slide_events, composited_path, + tmp_dir, total_duration, + ) + video_only_path = composited_path + # If there are audio-only tracks, overlay them if audio_files: logging.info(f'Merging {len(audio_files)} audio-only tracks...') diff --git a/mtslinker/webinar.py b/mtslinker/webinar.py index e5fa10b..ee4ecda 100644 --- a/mtslinker/webinar.py +++ b/mtslinker/webinar.py @@ -6,6 +6,7 @@ construct_json_data_url, fetch_json_data, download_chunks_parallel, + download_slide_images, ) from mtslinker.processor import compile_final_video, process_and_download_clips from mtslinker.utils import create_directory_if_not_exists @@ -23,14 +24,22 @@ def fetch_webinar_data(event_sessions: str, record_id: str, session_id=None, max directory = create_directory_if_not_exists(sanitized_name) output_video_path = os.path.join(directory, f'{sanitized_name}.mp4') - total_duration, chunks = process_and_download_clips(directory, json_data) + total_duration, chunks, slide_events = process_and_download_clips(directory, json_data) logging.info(f'Found {len(chunks)} chunks to download ({total_duration} sec total)') # Download all chunks in parallel downloaded_files = download_chunks_parallel(chunks, directory) logging.info(f'Downloaded {len(downloaded_files)} files, starting merge...') - compile_final_video(total_duration, downloaded_files, directory, output_video_path, max_duration) + # Download presentation slides if any + downloaded_slides = [] + if slide_events: + downloaded_slides = download_slide_images(slide_events, directory) + + compile_final_video( + total_duration, downloaded_files, directory, output_video_path, + max_duration, slide_events=downloaded_slides, + ) logging.info(f'Final video saved to {output_video_path}') return 1 From f1f7b8cb126efad703022da00479444ab00ae195 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 27 Mar 2026 08:52:40 +0000 Subject: [PATCH 08/65] Fix resolution detection: use largest segment, enforce 640x360 minimum Some recordings have tiny thumbnail-sized video segments (192x108) as the first file. The old code used the first segment's resolution for all normalization, resulting in a blurry output. Now scans all segments and picks the largest, with a 640x360 floor. --- mtslinker/processor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 75a2401..a4008fb 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -427,8 +427,15 @@ def compile_final_video( return # Determine target resolution from the first video segment - first_video = video_files[0][0] - target_w, target_h, target_pix_fmt = _get_video_params(first_video) + # Pick the largest resolution among video segments (some may be tiny thumbnails) + target_w, target_h, target_pix_fmt = 0, 0, 'yuv420p' + for vpath, _ in video_files: + w, h, pf = _get_video_params(vpath) + if w * h > target_w * target_h: + target_w, target_h, target_pix_fmt = w, h, pf + # Minimum 640x360 for reasonable quality + if target_w * target_h < 640 * 360: + target_w, target_h = 640, 360 logging.info(f'Target resolution: {target_w}x{target_h}, pix_fmt={target_pix_fmt}') # Sort by start time From e44dd00d37f83b030bd26a444ba2efb06720eade Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sat, 28 Mar 2026 08:59:14 +0000 Subject: [PATCH 09/65] Prefer presenter in dedup: use conf_id segment count as priority MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When multiple webcams overlap at the same timestamp, the old code kept the longest segment (often a random participant). Now tracks conference ID from the API and prefers the user with the most total segments across the recording — typically the presenter/instructor. Falls back to longest segment when conf_id is unavailable. --- mtslinker/downloader.py | 27 ++++++++------ mtslinker/processor.py | 81 ++++++++++++++++++++++++++--------------- 2 files changed, 66 insertions(+), 42 deletions(-) diff --git a/mtslinker/downloader.py b/mtslinker/downloader.py index 2c91859..6f38aba 100644 --- a/mtslinker/downloader.py +++ b/mtslinker/downloader.py @@ -175,33 +175,36 @@ def download_slide_images( def download_chunks_parallel( - chunks: List[Tuple[str, float]], + chunks: list, save_directory: str, max_workers: int = MAX_PARALLEL_DOWNLOADS, -) -> List[Tuple[str, float]]: +) -> list: """Download multiple chunks in parallel. Args: - chunks: List of (url, start_time) tuples. + chunks: List of (url, start_time) or (url, start_time, conf_id) tuples. save_directory: Directory to save files to. max_workers: Maximum number of parallel downloads. Returns: - List of (file_path, start_time) tuples in original order. + List of (file_path, start_time, conf_id) tuples in original order. """ results = [None] * len(chunks) - def _download(index, url, start_time): + def _download(index, url, start_time, conf_id): path = download_video_chunk(url, save_directory) - return index, path, start_time + return index, path, start_time, conf_id with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [ - executor.submit(_download, i, url, start_time) - for i, (url, start_time) in enumerate(chunks) - ] + futures = [] + for i, chunk in enumerate(chunks): + url, start_time = chunk[0], chunk[1] + conf_id = chunk[2] if len(chunk) > 2 else None + futures.append( + executor.submit(_download, i, url, start_time, conf_id) + ) for future in as_completed(futures): - idx, path, start_time = future.result() - results[idx] = (path, start_time) + idx, path, start_time, conf_id = future.result() + results[idx] = (path, start_time, conf_id) return results diff --git a/mtslinker/processor.py b/mtslinker/processor.py index a4008fb..6c7bfe3 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -187,7 +187,13 @@ def process_and_download_clips( # Media chunks (video/audio) if 'url' in data: - chunks.append((data['url'], event.get('relativeTime', 0))) + stream = data.get('stream', {}) + conf_id = None + if isinstance(stream, dict): + conf = stream.get('conference', {}) + if isinstance(conf, dict): + conf_id = conf.get('id') + chunks.append((data['url'], event.get('relativeTime', 0), conf_id)) # Presentation slide changes if event.get('module') == 'presentation.update': @@ -217,52 +223,65 @@ def process_and_download_clips( def _deduplicate_overlapping( - video_files: List[Tuple[str, float]], + video_files: list, ) -> List[Tuple[str, float]]: - """Remove overlapping video segments, keeping the longest per time window. + """Remove overlapping video segments, keeping the best per time window. - Recordings often have parallel streams (webcam + screen share) at the + Recordings often have parallel streams (multiple webcams) at the same timestamp. Laying them out sequentially inflates the duration. - This keeps only one segment per overlapping group — the longest one — - so the final timeline matches real elapsed time. + This keeps only one segment per overlapping group. + + Priority: prefer the conference (user) with the most total segments + (likely the presenter), then fall back to longest segment. """ if not video_files: return video_files - # Annotate with duration - annotated = [] # (path, start, duration, end) - for path, start in video_files: + # Count segments per conf_id to identify the "main" user + from collections import Counter + conf_counts = Counter() + for item in video_files: + conf_id = item[2] if len(item) > 2 else None + if conf_id: + conf_counts[conf_id] += 1 + + # Annotate with duration and conf_id + annotated = [] # (path, start, duration, end, conf_id) + for item in video_files: + path, start = item[0], item[1] + conf_id = item[2] if len(item) > 2 else None dur = _get_duration(path) - annotated.append((path, start, dur, start + dur)) + annotated.append((path, start, dur, start + dur, conf_id)) + + def _score(seg): + """Higher score = more preferred. Prefer main user, then longer.""" + _, _, dur, _, conf_id = seg + conf_rank = conf_counts.get(conf_id, 0) if conf_id else 0 + return (conf_rank, dur) - # Sort by start time, then longest first so we prefer longer segments - annotated.sort(key=lambda x: (x[1], -x[2])) + # Sort by start time, then best score first + annotated.sort(key=lambda x: (x[1], -_score(x)[0], -_score(x)[1])) - kept = [] # (path, start, duration, end) + kept = [] for seg in annotated: - path, start, dur, end = seg + path, start, dur, end, conf_id = seg if not kept: kept.append(seg) continue - _, prev_start, _, prev_end = kept[-1] + _, prev_start, _, prev_end, _ = kept[-1] if start >= prev_end - 0.5: - # No meaningful overlap — keep this segment kept.append(seg) else: - # Overlaps with the previous winner — keep whichever is longer - if dur > kept[-1][2]: + # Overlaps — keep the one with better score + if _score(seg) > _score(kept[-1]): logging.debug( - f'Dedup: replacing {kept[-1][2]:.1f}s segment at ' - f'{prev_start:.1f}s with {dur:.1f}s segment at {start:.1f}s' + f'Dedup: replacing {kept[-1][2]:.1f}s segment ' + f'(conf={kept[-1][4]}) with {dur:.1f}s segment ' + f'(conf={conf_id}, score={_score(seg)})' ) kept[-1] = seg - else: - logging.debug( - f'Dedup: skipping {dur:.1f}s segment at {start:.1f}s ' - f'(covered by {kept[-1][2]:.1f}s segment at {prev_start:.1f}s)' - ) original = len(video_files) deduped = len(kept) @@ -270,7 +289,7 @@ def _deduplicate_overlapping( logging.info(f'Dedup: {original} -> {deduped} segments ' f'(removed {original - deduped} overlapping)') - return [(path, start) for path, start, dur, end in kept] + return [(path, start) for path, start, dur, end, conf_id in kept] def _composite_slides( @@ -405,16 +424,18 @@ def compile_final_video( """ _check_ffmpeg() - video_files = [] # (path, start_time) + video_files = [] # (path, start_time, conf_id) audio_files = [] # (path, start_time) skipped = 0 - for file_path, start_time in downloaded_files: + for item in downloaded_files: + file_path, start_time = item[0], item[1] + conf_id = item[2] if len(item) > 2 else None if not _is_valid_media(file_path): skipped += 1 continue if _has_video_stream(file_path): - video_files.append((file_path, start_time)) + video_files.append((file_path, start_time, conf_id)) else: audio_files.append((file_path, start_time)) @@ -429,7 +450,7 @@ def compile_final_video( # Determine target resolution from the first video segment # Pick the largest resolution among video segments (some may be tiny thumbnails) target_w, target_h, target_pix_fmt = 0, 0, 'yuv420p' - for vpath, _ in video_files: + for vpath, *_ in video_files: w, h, pf = _get_video_params(vpath) if w * h > target_w * target_h: target_w, target_h, target_pix_fmt = w, h, pf From 4d0b10053005951292213d35c2d19eee6f0bf225 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sat, 28 Mar 2026 14:58:55 +0000 Subject: [PATCH 10/65] Fix stuck slide segment encoding for long durations The -loop 1 -framerate 1 -t approach could produce millions of frames for long-duration slides (e.g., last slide staying up for 3 hours), causing ffmpeg to spin for hours and write gigabytes. Now uses -frames:v to strictly cap frame count to match duration at 1fps. --- mtslinker/processor.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 6c7bfe3..5d12464 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -326,18 +326,23 @@ def _composite_slides( continue seg_path = os.path.join(slides_dir, f'seg_{i}.mp4') + # Use -frames:v to strictly limit frame count at 1fps. + # This avoids runaway encoding for long durations. + n_frames = max(1, int(duration)) _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', - '-loop', '1', '-framerate', '1', '-t', str(duration), + '-loop', '1', '-framerate', '1', '-i', se['local_path'], '-vf', f'scale={SLIDE_W}:{SLIDE_H}:force_original_aspect_ratio=decrease,' f'pad={SLIDE_W}:{SLIDE_H}:(ow-iw)/2:(oh-ih)/2:white', '-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23', - '-pix_fmt', 'yuv420p', '-r', '1', + '-pix_fmt', 'yuv420p', + '-r', '1', '-frames:v', str(n_frames), seg_path, ], - description=f'slide segment {i+1}/{len(slide_events)}', + description=f'slide segment {i+1}/{len(slide_events)} ' + f'({n_frames} frames, {duration:.0f}s)', ) slide_segments.append(seg_path) From f95646e234cc506132607f1a89b0dcac7be96036 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sat, 28 Mar 2026 15:05:04 +0000 Subject: [PATCH 11/65] Add NVENC GPU encoding support with automatic detection Detects h264_nvenc at startup and uses it for all encoding steps if available. Falls back to libx264 CPU encoding if no GPU. Massively reduces CPU load and encoding time on systems with NVIDIA GPUs, while keeping the CPU cool. --- mtslinker/processor.py | 51 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 5d12464..9a4a34e 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -8,6 +8,47 @@ AUDIO_MERGE_BATCH_SIZE = 8 +def _has_nvenc() -> bool: + """Check if NVIDIA NVENC hardware encoder is available.""" + try: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-f', 'lavfi', '-i', 'nullsrc=s=64x64:d=0.1', + '-c:v', 'h264_nvenc', '-f', 'null', '-'], + capture_output=True, timeout=10, + ) + return result.returncode == 0 + except Exception: + return False + + +# Detected once at import time +_NVENC_AVAILABLE = None + + +def _get_video_encoder() -> list: + """Return ffmpeg video encoder args, preferring NVENC if available.""" + global _NVENC_AVAILABLE + if _NVENC_AVAILABLE is None: + _NVENC_AVAILABLE = _has_nvenc() + if _NVENC_AVAILABLE: + logging.info('NVENC GPU encoder detected, using h264_nvenc') + else: + logging.info('No NVENC, using libx264 CPU encoder') + if _NVENC_AVAILABLE: + return ['-c:v', 'h264_nvenc', '-preset', 'p4', '-cq', '23'] + return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] + + +def _get_video_encoder_fast() -> list: + """Return fast encoder args for simple content (slides, gaps).""" + global _NVENC_AVAILABLE + if _NVENC_AVAILABLE is None: + _NVENC_AVAILABLE = _has_nvenc() + if _NVENC_AVAILABLE: + return ['-c:v', 'h264_nvenc', '-preset', 'p1', '-cq', '28'] + return ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23'] + + def _check_ffmpeg(): """Check that ffmpeg and ffprobe are available.""" for tool in ('ffmpeg', 'ffprobe'): @@ -106,7 +147,7 @@ def _generate_black_segment(output_path: str, duration: float, '-f', 'lavfi', '-i', f'color=c=black:s={width}x{height}:d={duration}:r=25', '-f', 'lavfi', '-i', f'anullsrc=r=44100:cl=stereo', '-t', str(duration), - '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'stillimage', + *_get_video_encoder_fast(), '-pix_fmt', pix_fmt, '-c:a', 'aac', '-b:a', '128k', '-shortest', @@ -151,7 +192,7 @@ def _normalize_segment(input_path: str, output_path: str, f'pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black,' f'setsar=1', '-pix_fmt', pix_fmt, - '-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '18', + *_get_video_encoder_fast(), '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', '-r', '25', output_path, @@ -336,7 +377,7 @@ def _composite_slides( '-i', se['local_path'], '-vf', f'scale={SLIDE_W}:{SLIDE_H}:force_original_aspect_ratio=decrease,' f'pad={SLIDE_W}:{SLIDE_H}:(ow-iw)/2:(oh-ih)/2:white', - '-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23', + *_get_video_encoder_fast(), '-pix_fmt', 'yuv420p', '-r', '1', '-frames:v', str(n_frames), seg_path, @@ -355,7 +396,7 @@ def _composite_slides( 'ffmpeg', '-y', '-v', 'error', '-f', 'lavfi', '-i', f'color=c=black:s={SLIDE_W}x{SLIDE_H}:d={first_time}:r=1', - '-c:v', 'libx264', '-preset', 'ultrafast', + *_get_video_encoder_fast(), '-pix_fmt', 'yuv420p', leader_path, ], @@ -395,7 +436,7 @@ def _composite_slides( '-i', video_path, '-filter_complex', filter_graph, '-map', '[out]', '-map', '1:a?', - '-c:v', 'libx264', '-preset', 'fast', '-crf', '23', + *_get_video_encoder(), '-c:a', 'copy', '-r', '25', '-shortest', From c3fa7edfaf995633316c9919e491c8971a28399d Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 31 Mar 2026 23:16:57 +0000 Subject: [PATCH 12/65] Use CUDA GPU filters for overlay compositing when available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The overlay step was CPU-bound (97°C). Now uses hwupload_cuda, scale_cuda, and overlay_cuda to do the compositing entirely on GPU. Falls back to CPU filters if CUDA overlay is not available. --- mtslinker/processor.py | 56 ++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 9a4a34e..844c85a 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -21,19 +21,40 @@ def _has_nvenc() -> bool: return False +def _has_cuda_overlay() -> bool: + """Check if ffmpeg has CUDA overlay filter support.""" + try: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-filters'], + capture_output=True, text=True, timeout=10, + ) + return 'overlay_cuda' in result.stdout + except Exception: + return False + + # Detected once at import time _NVENC_AVAILABLE = None +_CUDA_OVERLAY_AVAILABLE = None -def _get_video_encoder() -> list: - """Return ffmpeg video encoder args, preferring NVENC if available.""" - global _NVENC_AVAILABLE +def _detect_gpu(): + """Detect GPU capabilities once.""" + global _NVENC_AVAILABLE, _CUDA_OVERLAY_AVAILABLE if _NVENC_AVAILABLE is None: _NVENC_AVAILABLE = _has_nvenc() - if _NVENC_AVAILABLE: - logging.info('NVENC GPU encoder detected, using h264_nvenc') + _CUDA_OVERLAY_AVAILABLE = _has_cuda_overlay() if _NVENC_AVAILABLE else False + if _CUDA_OVERLAY_AVAILABLE: + logging.info('CUDA overlay + NVENC detected, using full GPU pipeline') + elif _NVENC_AVAILABLE: + logging.info('NVENC detected (no CUDA overlay), using GPU encoder only') else: - logging.info('No NVENC, using libx264 CPU encoder') + logging.info('No GPU support, using CPU pipeline') + + +def _get_video_encoder() -> list: + """Return ffmpeg video encoder args, preferring NVENC if available.""" + _detect_gpu() if _NVENC_AVAILABLE: return ['-c:v', 'h264_nvenc', '-preset', 'p4', '-cq', '23'] return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] @@ -422,13 +443,22 @@ def _composite_slides( logging.info('Slide track created') # Step 2: Combine slide track + webcam into final layout. - # Simple 2-input overlay: slide on left, webcam scaled to top-right. - filter_graph = ( - f'[1:v]scale={CAM_W}:{CAM_H}:force_original_aspect_ratio=decrease,' - f'pad={CAM_W}:{CAM_H}:(ow-iw)/2:(oh-ih)/2:black[webcam];' - f'[0:v]pad={CANVAS_W}:{CANVAS_H}:0:0:black[padded];' - f'[padded][webcam]overlay={SLIDE_W}:0[out]' - ) + _detect_gpu() + if _CUDA_OVERLAY_AVAILABLE: + # Full GPU pipeline: upload to CUDA, scale/pad/overlay on GPU + filter_graph = ( + f'[1:v]hwupload_cuda,scale_cuda={CAM_W}:{CAM_H}[webcam_gpu];' + f'[0:v]hwupload_cuda,scale_cuda={CANVAS_W}:{CANVAS_H}[padded_gpu];' + f'[padded_gpu][webcam_gpu]overlay_cuda={SLIDE_W}:0[out]' + ) + else: + # CPU fallback + filter_graph = ( + f'[1:v]scale={CAM_W}:{CAM_H}:force_original_aspect_ratio=decrease,' + f'pad={CAM_W}:{CAM_H}:(ow-iw)/2:(oh-ih)/2:black[webcam];' + f'[0:v]pad={CANVAS_W}:{CANVAS_H}:0:0:black[padded];' + f'[padded][webcam]overlay={SLIDE_W}:0[out]' + ) cmd = [ 'ffmpeg', '-y', '-v', 'warning', From 5e71b2d931243ec028d454c057832729d60393df Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 2 Apr 2026 00:22:55 +0000 Subject: [PATCH 13/65] Fix webcam framerate + add grid layout for multi-webcam recordings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes: 1. Swap inputs in slide compositing so webcam (25fps) drives the output frame clock instead of the slide track (1fps). Fixes choppy webcam playback in presentation videos. 2. For recordings without presentation slides that have multiple concurrent webcams (ПЗ sessions), composite all active webcams into a grid layout using xstack instead of discarding all but one. Grid size adapts to the number of concurrent webcams (2x1, 2x2, 3x3 etc). Audio from all participants is mixed. --- mtslinker/processor.py | 263 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 243 insertions(+), 20 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 844c85a..a4c2c17 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1,5 +1,6 @@ import json import logging +import math import os import shutil import subprocess @@ -223,6 +224,213 @@ def _normalize_segment(input_path: str, output_path: str, return output_path +def _compute_grid(n: int) -> Tuple[int, int]: + """Return (cols, rows) for a grid holding n items.""" + cols = math.ceil(math.sqrt(n)) + rows = math.ceil(n / cols) + return cols, rows + + +def _composite_grid( + active_segments: list, + duration: float, + output_path: str, + target_w: int, + target_h: int, +) -> str: + """Create a grid video from multiple simultaneous webcam segments. + + Args: + active_segments: list of (path, offset_within_file) tuples + duration: length of this time window + output_path: where to write + target_w, target_h: output resolution + """ + n = len(active_segments) + cols, rows = _compute_grid(n) + cell_w = target_w // cols + cell_h = target_h // rows + + inputs = [] + filter_parts = [] + labels = [] + + for i, (path, offset) in enumerate(active_segments): + inputs.extend(['-ss', str(offset), '-i', path]) + label = f'v{i}' + filter_parts.append( + f'[{i}:v]scale={cell_w}:{cell_h}:force_original_aspect_ratio=decrease,' + f'pad={cell_w}:{cell_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[{label}]' + ) + labels.append(f'[{label}]') + + # Pad with black cells if needed + total_cells = cols * rows + for i in range(n, total_cells): + inputs.extend(['-f', 'lavfi', '-i', + f'color=black:s={cell_w}x{cell_h}:d={duration}:r=25']) + label = f'v{i}' + labels.append(f'[{len(active_segments) + i - n}:v]') + # lavfi inputs don't need scaling, rename + idx = len(active_segments) + i - n + labels[-1] = f'[{idx}:v]' + + # Build xstack layout string: x_y positions + layout_parts = [] + for i in range(total_cells): + c = i % cols + r = i // cols + layout_parts.append(f'{c * cell_w}_{r * cell_h}') + layout = '|'.join(layout_parts) + + filter_graph = ';'.join(filter_parts) + if filter_graph: + filter_graph += ';' + filter_graph += ( + ''.join(labels) + + f'xstack=inputs={total_cells}:layout={layout}[out]' + ) + + # Mix all audio streams + audio_labels = ''.join(f'[{i}:a]' for i in range(n)) + if n > 1: + filter_graph += f';{audio_labels}amix=inputs={n}:duration=longest:normalize=0[aout]' + audio_map = ['-map', '[aout]'] + else: + audio_map = ['-map', '0:a?'] + + cmd = [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-t', str(duration), + '-filter_complex', filter_graph, + '-map', '[out]', *audio_map, + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', + '-r', '25', + output_path, + ] + _run_ffmpeg(cmd, description=f'grid {n} webcams, {duration:.0f}s') + return output_path + + +def _build_grid_segments( + video_files: list, + tmp_dir: str, + target_w: int, + target_h: int, + total_duration: float, +) -> List[Tuple[str, float]]: + """Build grid-composited segments from overlapping webcam streams. + + Instead of deduplicating, composites all concurrent webcams into a grid. + Returns a list of (segment_path, start_time) ready for concatenation. + """ + # Annotate with duration + annotated = [] + for item in video_files: + path, start = item[0], item[1] + dur = _get_duration(path) + if dur > 0: + annotated.append((path, start, dur, start + dur)) + + if not annotated: + return [] + + # Collect all event times (segment starts and ends) + events = set() + for _, start, _, end in annotated: + events.add(start) + events.add(end) + events.add(total_duration) + event_times = sorted(events) + + # Merge adjacent windows with the same active set + grid_dir = os.path.join(tmp_dir, 'grid_segments') + os.makedirs(grid_dir, exist_ok=True) + + result = [] + prev_active = None + window_start = None + + for i in range(len(event_times) - 1): + t_start = event_times[i] + t_end = event_times[i + 1] + if t_end - t_start < 0.1: + continue + + # Find active segments in this window + active = [] + for path, seg_start, dur, seg_end in annotated: + if seg_start < t_end and seg_end > t_start: + offset = max(0, t_start - seg_start) + active.append((path, offset)) + + active_key = tuple(a[0] for a in active) + + # Merge with previous window if same active set + if active_key == prev_active and window_start is not None: + continue # will be handled when active set changes + + # Emit previous merged window + if prev_active is not None and window_start is not None: + merged_end = t_start + merged_dur = merged_end - window_start + if merged_dur > 0.1: + _emit_grid_window( + prev_active_segs, merged_dur, window_start, + grid_dir, len(result), target_w, target_h, result, + ) + + window_start = t_start + prev_active = active_key + prev_active_segs = active + + # Emit final window + if prev_active is not None and window_start is not None: + merged_end = event_times[-1] + merged_dur = merged_end - window_start + if merged_dur > 0.1: + _emit_grid_window( + prev_active_segs, merged_dur, window_start, + grid_dir, len(result), target_w, target_h, result, + ) + + logging.info(f'Grid: built {len(result)} segments from ' + f'{len(annotated)} webcam streams') + return result + + +def _emit_grid_window(active, duration, start_time, grid_dir, idx, + target_w, target_h, result): + """Helper to emit a single grid window segment.""" + if not active: + return + seg_path = os.path.join(grid_dir, f'grid_{idx}.mp4') + if len(active) == 1: + # Single webcam — just extract the slice + path, offset = active[0] + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(offset), '-i', path, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'grid window {idx} (1 webcam, {duration:.0f}s)', + ) + else: + _composite_grid(active, duration, seg_path, target_w, target_h) + result.append((seg_path, start_time)) + + def process_and_download_clips( directory: str, json_data: Dict ) -> Tuple[float, List[Tuple[str, float]], List[Dict]]: @@ -443,29 +651,30 @@ def _composite_slides( logging.info('Slide track created') # Step 2: Combine slide track + webcam into final layout. + # Webcam is input 0 (25fps, drives output frame rate). + # Slide track is input 1 (1fps, overlaid on left). _detect_gpu() if _CUDA_OVERLAY_AVAILABLE: - # Full GPU pipeline: upload to CUDA, scale/pad/overlay on GPU filter_graph = ( - f'[1:v]hwupload_cuda,scale_cuda={CAM_W}:{CAM_H}[webcam_gpu];' - f'[0:v]hwupload_cuda,scale_cuda={CANVAS_W}:{CANVAS_H}[padded_gpu];' - f'[padded_gpu][webcam_gpu]overlay_cuda={SLIDE_W}:0[out]' + f'[0:v]hwupload_cuda,scale_cuda={CAM_W}:{CAM_H}[webcam_gpu];' + f'[1:v]hwupload_cuda,scale_cuda={CANVAS_W}:{CANVAS_H}[slide_gpu];' + f'[slide_gpu][webcam_gpu]overlay_cuda={SLIDE_W}:0[out]' ) else: - # CPU fallback filter_graph = ( - f'[1:v]scale={CAM_W}:{CAM_H}:force_original_aspect_ratio=decrease,' - f'pad={CAM_W}:{CAM_H}:(ow-iw)/2:(oh-ih)/2:black[webcam];' - f'[0:v]pad={CANVAS_W}:{CANVAS_H}:0:0:black[padded];' - f'[padded][webcam]overlay={SLIDE_W}:0[out]' + f'[0:v]scale={CAM_W}:{CAM_H}:force_original_aspect_ratio=decrease,' + f'pad={CANVAS_W}:{CANVAS_H}:{SLIDE_W}:0:black[base];' + f'[1:v]scale={SLIDE_W}:{SLIDE_H}:force_original_aspect_ratio=decrease,' + f'pad={SLIDE_W}:{SLIDE_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' + f'[base][slide]overlay=0:0[out]' ) cmd = [ 'ffmpeg', '-y', '-v', 'warning', - '-i', slide_track_path, '-i', video_path, + '-i', slide_track_path, '-filter_complex', filter_graph, - '-map', '[out]', '-map', '1:a?', + '-map', '[out]', '-map', '0:a?', *_get_video_encoder(), '-c:a', 'copy', '-r', '25', @@ -538,19 +747,33 @@ def compile_final_video( # Sort by start time video_files.sort(key=lambda x: x[1]) - # Deduplicate overlapping segments: recordings often have parallel - # streams (webcam + screen share) running at the same time. If we - # concatenate them all sequentially the output is 3-4x too long. - # Strategy: walk through sorted segments; when a new segment starts - # before the current winner ends, keep whichever is longer and - # discard the shorter one. - video_files = _deduplicate_overlapping(video_files) - logging.info(f'After dedup: {len(video_files)} non-overlapping video segments') - # Build the list of segments (normalized videos + gap fillers) tmp_dir = os.path.join(directory, '_tmp_ffmpeg') os.makedirs(tmp_dir, exist_ok=True) + # Check for overlapping segments (multiple concurrent webcams) + has_overlaps = False + annotated_check = sorted( + [(item[0], item[1], _get_duration(item[0])) for item in video_files], + key=lambda x: x[1], + ) + for i in range(1, len(annotated_check)): + prev_end = annotated_check[i-1][1] + annotated_check[i-1][2] + if annotated_check[i][1] < prev_end - 0.5: + has_overlaps = True + break + + if has_overlaps and not slide_events: + # Grid layout: composite all concurrent webcams into a grid + logging.info('Multiple concurrent webcams detected, building grid layout') + video_files = _build_grid_segments( + video_files, tmp_dir, target_w, target_h, total_duration, + ) + else: + # Deduplicate overlapping segments (keep presenter for slide videos) + video_files = _deduplicate_overlapping(video_files) + logging.info(f'After dedup/grid: {len(video_files)} segments') + concat_segments = [] current_time = 0.0 From d24229e5dd19e9951ae1125a75cd86e8edbec1fc Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 2 Apr 2026 22:51:41 +0000 Subject: [PATCH 14/65] fix: skip audio in grid composite - audio handled by separate merge pipeline Webcam inputs may lack audio tracks, causing ffmpeg to fail with 'Stream specifier :a matches no streams'. Since _merge_audio_tracks handles all audio separately, the grid step should output video only. --- mtslinker/processor.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index a4c2c17..7c859b2 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -291,23 +291,16 @@ def _composite_grid( + f'xstack=inputs={total_cells}:layout={layout}[out]' ) - # Mix all audio streams - audio_labels = ''.join(f'[{i}:a]' for i in range(n)) - if n > 1: - filter_graph += f';{audio_labels}amix=inputs={n}:duration=longest:normalize=0[aout]' - audio_map = ['-map', '[aout]'] - else: - audio_map = ['-map', '0:a?'] - + # Audio is handled separately by _merge_audio_tracks — output video only. cmd = [ 'ffmpeg', '-y', '-v', 'error', *inputs, '-t', str(duration), '-filter_complex', filter_graph, - '-map', '[out]', *audio_map, + '-map', '[out]', + '-an', *_get_video_encoder_fast(), '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-r', '25', output_path, ] From 3ed353945cf20b1eac6dfe0ff0262340cb76c481 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 2 Apr 2026 23:17:44 +0000 Subject: [PATCH 15/65] fix: dedup by total duration instead of segment count The old scoring picked the conference with the most segments, which favored participants toggling their cameras (many short segments) over the presenter (few long segments). Also had a window-shrinking bug where replacing a long segment with a short higher-ranked one let subsequent segments leak through. New approach: identify the main conference by total recorded duration, keep its segments, and fill gaps from other conferences. --- mtslinker/processor.py | 95 ++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 37 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 7c859b2..b4b64a4 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -488,27 +488,24 @@ def process_and_download_clips( def _deduplicate_overlapping( video_files: list, ) -> List[Tuple[str, float]]: - """Remove overlapping video segments, keeping the best per time window. + """Remove overlapping video segments, keeping one per time window. Recordings often have parallel streams (multiple webcams) at the same timestamp. Laying them out sequentially inflates the duration. - This keeps only one segment per overlapping group. - Priority: prefer the conference (user) with the most total segments - (likely the presenter), then fall back to longest segment. + Strategy: + 1. Pick the "main" conference — the one with the most total recorded + duration (the presenter typically has a few long segments, while + participants toggle cameras creating many short ones). + 2. Keep only segments from the main conference. + 3. For time gaps where the main conference has no video, fill in + with the longest available segment from any other conference. """ if not video_files: return video_files - # Count segments per conf_id to identify the "main" user - from collections import Counter - conf_counts = Counter() - for item in video_files: - conf_id = item[2] if len(item) > 2 else None - if conf_id: - conf_counts[conf_id] += 1 - # Annotate with duration and conf_id + from collections import defaultdict annotated = [] # (path, start, duration, end, conf_id) for item in video_files: path, start = item[0], item[1] @@ -516,43 +513,67 @@ def _deduplicate_overlapping( dur = _get_duration(path) annotated.append((path, start, dur, start + dur, conf_id)) - def _score(seg): - """Higher score = more preferred. Prefer main user, then longer.""" - _, _, dur, _, conf_id = seg - conf_rank = conf_counts.get(conf_id, 0) if conf_id else 0 - return (conf_rank, dur) - - # Sort by start time, then best score first - annotated.sort(key=lambda x: (x[1], -_score(x)[0], -_score(x)[1])) + # Score each conference by total duration (not segment count) + conf_total_dur = defaultdict(float) + for _, _, dur, _, conf_id in annotated: + if conf_id: + conf_total_dur[conf_id] += dur - kept = [] - for seg in annotated: - path, start, dur, end, conf_id = seg - if not kept: - kept.append(seg) - continue + if conf_total_dur: + main_conf = max(conf_total_dur, key=conf_total_dur.get) + logging.info( + f'Dedup: main conference {main_conf} ' + f'({conf_total_dur[main_conf]:.0f}s total from ' + f'{sum(1 for x in annotated if x[4] == main_conf)} segments)' + ) + else: + main_conf = None - _, prev_start, _, prev_end, _ = kept[-1] + # Separate main conference segments from others + main_segs = sorted( + [s for s in annotated if s[4] == main_conf], + key=lambda x: x[1], + ) + other_segs = sorted( + [s for s in annotated if s[4] != main_conf], + key=lambda x: x[1], + ) - if start >= prev_end - 0.5: + # Build timeline from main conference segments (merge overlapping) + kept = [] + for seg in main_segs: + if not kept or seg[1] >= kept[-1][3] - 0.5: kept.append(seg) else: - # Overlaps — keep the one with better score - if _score(seg) > _score(kept[-1]): - logging.debug( - f'Dedup: replacing {kept[-1][2]:.1f}s segment ' - f'(conf={kept[-1][4]}) with {dur:.1f}s segment ' - f'(conf={conf_id}, score={_score(seg)})' - ) + # Overlapping main segments — keep the longer one + if seg[2] > kept[-1][2]: kept[-1] = seg + # Fill gaps with best available from other conferences + filled = [] + for i, seg in enumerate(kept): + gap_start = kept[i - 1][3] if i > 0 else 0 + gap_end = seg[1] + if gap_end - gap_start > 1.0: + # Find the longest other-conference segment covering this gap + best = None + for other in other_segs: + # Segment must overlap the gap + if other[3] <= gap_start or other[1] >= gap_end: + continue + if best is None or other[2] > best[2]: + best = other + if best: + filled.append(best) + filled.append(seg) + original = len(video_files) - deduped = len(kept) + deduped = len(filled) if original != deduped: logging.info(f'Dedup: {original} -> {deduped} segments ' f'(removed {original - deduped} overlapping)') - return [(path, start) for path, start, dur, end, conf_id in kept] + return [(path, start) for path, start, dur, end, conf_id in filled] def _composite_slides( From 5cec9f92c64140894a51f77fcd9cbe05e3eb56b6 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 3 Apr 2026 00:00:01 +0000 Subject: [PATCH 16/65] fix: prefer ADMIN user webcam in dedup, pass is_admin through pipeline Extracts ADMIN role from userlist events, maps to conference IDs via conference.add events, and passes is_admin flag through the download pipeline. Dedup now prefers ADMIN conferences (the presenter), falling back to total duration when no admin is found. Also fixes download_chunks_parallel to preserve the is_admin flag. --- mtslinker/downloader.py | 15 ++++--- mtslinker/processor.py | 98 ++++++++++++++++++++++++++++++++++------- 2 files changed, 91 insertions(+), 22 deletions(-) diff --git a/mtslinker/downloader.py b/mtslinker/downloader.py index 6f38aba..7819830 100644 --- a/mtslinker/downloader.py +++ b/mtslinker/downloader.py @@ -182,29 +182,30 @@ def download_chunks_parallel( """Download multiple chunks in parallel. Args: - chunks: List of (url, start_time) or (url, start_time, conf_id) tuples. + chunks: List of (url, start_time[, conf_id[, is_admin]]) tuples. save_directory: Directory to save files to. max_workers: Maximum number of parallel downloads. Returns: - List of (file_path, start_time, conf_id) tuples in original order. + List of (file_path, start_time, conf_id, is_admin) tuples in original order. """ results = [None] * len(chunks) - def _download(index, url, start_time, conf_id): + def _download(index, url, start_time, conf_id, is_admin): path = download_video_chunk(url, save_directory) - return index, path, start_time, conf_id + return index, path, start_time, conf_id, is_admin with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [] for i, chunk in enumerate(chunks): url, start_time = chunk[0], chunk[1] conf_id = chunk[2] if len(chunk) > 2 else None + is_admin = chunk[3] if len(chunk) > 3 else False futures.append( - executor.submit(_download, i, url, start_time, conf_id) + executor.submit(_download, i, url, start_time, conf_id, is_admin) ) for future in as_completed(futures): - idx, path, start_time, conf_id = future.result() - results[idx] = (path, start_time, conf_id) + idx, path, start_time, conf_id, is_admin = future.result() + results[idx] = (path, start_time, conf_id, is_admin) return results diff --git a/mtslinker/processor.py b/mtslinker/processor.py index b4b64a4..7d22f7a 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -424,13 +424,62 @@ def _emit_grid_window(active, duration, start_time, grid_dir, idx, result.append((seg_path, start_time)) +def _extract_admin_conf_ids(json_data: Dict) -> set: + """Find conference IDs belonging to ADMIN users. + + Uses userlist events to find ADMIN user IDs, then maps them to + conference IDs via conference.add events. + """ + admin_user_ids = set() + user_to_conf = {} # user_id -> set of conf_ids + + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data_list = event.get('data', []) + if isinstance(data_list, dict): + data_list = [data_list] + if not isinstance(data_list, list): + continue + + for d in data_list: + if not isinstance(d, dict): + continue + + if 'userlist' in module: + role = d.get('role', '') + user = d.get('user', {}) + if isinstance(user, dict) and role == 'ADMIN': + uid = user.get('id') + if uid: + admin_user_ids.add(uid) + + if module == 'conference.add': + user = d.get('user', {}) + if isinstance(user, dict): + uid = user.get('id') + cid = d.get('id') + if uid and cid: + user_to_conf.setdefault(uid, set()).add(cid) + + admin_confs = set() + for uid in admin_user_ids: + admin_confs.update(user_to_conf.get(uid, set())) + + if admin_confs: + logging.info(f'Found {len(admin_confs)} conference(s) from ADMIN users') + + return admin_confs + + def process_and_download_clips( directory: str, json_data: Dict ) -> Tuple[float, List[Tuple[str, float]], List[Dict]]: """Extract chunk URLs, start times, and presentation slides from the API JSON. Returns: - (total_duration, [(url, start_time), ...], [slide_event, ...]) + (total_duration, [(url, start_time, conf_id), ...], [slide_event, ...]) Each slide_event is: {"time": float, "slide_number": int, "slide_url": str} @@ -439,6 +488,8 @@ def process_and_download_clips( if not total_duration: raise ValueError('Duration not found in JSON data.') + admin_conf_ids = _extract_admin_conf_ids(json_data) + chunks = [] slide_events = [] for event in json_data.get('eventLogs', []): @@ -473,6 +524,11 @@ def process_and_download_clips( 'slide_url': slide_url, }) + # Tag chunks from admin conferences for dedup priority + tagged_chunks = [] + for url, start, conf_id in chunks: + tagged_chunks.append((url, start, conf_id, conf_id in admin_conf_ids)) + # Deduplicate consecutive identical slides deduped_slides = [] for se in slide_events: @@ -482,7 +538,7 @@ def process_and_download_clips( if deduped_slides: logging.info(f'Found {len(deduped_slides)} presentation slide changes') - return total_duration, chunks, deduped_slides + return total_duration, tagged_chunks, deduped_slides def _deduplicate_overlapping( @@ -494,9 +550,8 @@ def _deduplicate_overlapping( same timestamp. Laying them out sequentially inflates the duration. Strategy: - 1. Pick the "main" conference — the one with the most total recorded - duration (the presenter typically has a few long segments, while - participants toggle cameras creating many short ones). + 1. Pick the "main" conference: prefer ADMIN user conferences, + then fall back to the one with the most total recorded duration. 2. Keep only segments from the main conference. 3. For time gaps where the main conference has no video, fill in with the longest available segment from any other conference. @@ -504,22 +559,35 @@ def _deduplicate_overlapping( if not video_files: return video_files - # Annotate with duration and conf_id + # Annotate with duration, conf_id, and is_admin from collections import defaultdict - annotated = [] # (path, start, duration, end, conf_id) + annotated = [] # (path, start, duration, end, conf_id, is_admin) for item in video_files: path, start = item[0], item[1] conf_id = item[2] if len(item) > 2 else None + is_admin = item[3] if len(item) > 3 else False dur = _get_duration(path) - annotated.append((path, start, dur, start + dur, conf_id)) + annotated.append((path, start, dur, start + dur, conf_id, is_admin)) - # Score each conference by total duration (not segment count) + # Score each conference by total duration conf_total_dur = defaultdict(float) - for _, _, dur, _, conf_id in annotated: + conf_is_admin = {} + for _, _, dur, _, conf_id, is_admin in annotated: if conf_id: conf_total_dur[conf_id] += dur + if is_admin: + conf_is_admin[conf_id] = True - if conf_total_dur: + # Pick main conference: ADMIN with most duration, else any with most duration + admin_confs = {c for c in conf_total_dur if conf_is_admin.get(c)} + if admin_confs: + main_conf = max(admin_confs, key=conf_total_dur.get) + logging.info( + f'Dedup: main conference {main_conf} (ADMIN, ' + f'{conf_total_dur[main_conf]:.0f}s total from ' + f'{sum(1 for x in annotated if x[4] == main_conf)} segments)' + ) + elif conf_total_dur: main_conf = max(conf_total_dur, key=conf_total_dur.get) logging.info( f'Dedup: main conference {main_conf} ' @@ -558,7 +626,6 @@ def _deduplicate_overlapping( # Find the longest other-conference segment covering this gap best = None for other in other_segs: - # Segment must overlap the gap if other[3] <= gap_start or other[1] >= gap_end: continue if best is None or other[2] > best[2]: @@ -573,7 +640,7 @@ def _deduplicate_overlapping( logging.info(f'Dedup: {original} -> {deduped} segments ' f'(removed {original - deduped} overlapping)') - return [(path, start) for path, start, dur, end, conf_id in filled] + return [(path, start) for path, start, dur, end, conf_id, is_admin in filled] def _composite_slides( @@ -723,18 +790,19 @@ def compile_final_video( """ _check_ffmpeg() - video_files = [] # (path, start_time, conf_id) + video_files = [] # (path, start_time, conf_id, is_admin) audio_files = [] # (path, start_time) skipped = 0 for item in downloaded_files: file_path, start_time = item[0], item[1] conf_id = item[2] if len(item) > 2 else None + is_admin = item[3] if len(item) > 3 else False if not _is_valid_media(file_path): skipped += 1 continue if _has_video_stream(file_path): - video_files.append((file_path, start_time, conf_id)) + video_files.append((file_path, start_time, conf_id, is_admin)) else: audio_files.append((file_path, start_time)) From 3ae30f7d35e79348d3b5b1fda026e2b364c5af0b Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 3 Apr 2026 08:02:23 +0000 Subject: [PATCH 17/65] fix: prevent duration inflation from overlapping segments + improve webcam layout Dedup gap-fill: clamp "other" conference segments to actual gap boundaries instead of using raw file duration, preventing timeline overflow. Compile: skip segments starting before current_time (safety net for overlaps), and truncate segments via -t so they can't overflow into the next segment. Slide composite: scale webcam proportionally to 320px wide (was fixed 320x180), so portrait webcams render at a usable size instead of being squished. --- mtslinker/processor.py | 66 +++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 7d22f7a..bda1740 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -204,8 +204,14 @@ def _ensure_audio_stream(input_path: str, output_path: str) -> str: def _normalize_segment(input_path: str, output_path: str, - width: int, height: int, pix_fmt: str) -> str: - """Re-encode a segment to a common format for reliable concatenation.""" + width: int, height: int, pix_fmt: str, + max_duration: float = 0) -> str: + """Re-encode a segment to a common format for reliable concatenation. + + Args: + max_duration: If > 0, truncate the output to this many seconds. + """ + duration_args = ['-t', str(max_duration)] if max_duration > 0 else [] _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', @@ -217,6 +223,7 @@ def _normalize_segment(input_path: str, output_path: str, *_get_video_encoder_fast(), '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', '-r', '25', + *duration_args, output_path, ], description=f'normalize segment {os.path.basename(input_path)}', @@ -628,8 +635,15 @@ def _deduplicate_overlapping( for other in other_segs: if other[3] <= gap_start or other[1] >= gap_end: continue - if best is None or other[2] > best[2]: - best = other + # Compute how much of this segment actually covers the gap + overlap_start = max(other[1], gap_start) + overlap_end = min(other[3], gap_end) + overlap_dur = overlap_end - overlap_start + if overlap_dur <= 0: + continue + if best is None or overlap_dur > best[2]: + best = (other[0], overlap_start, overlap_dur, + overlap_end, other[4], other[5]) if best: filled.append(best) filled.append(seg) @@ -658,8 +672,8 @@ def _composite_slides( - Right 320px, below webcam: black """ CANVAS_W, CANVAS_H = 1280, 720 - SLIDE_W, SLIDE_H = 960, 720 - CAM_W, CAM_H = 320, 180 + SLIDE_W, SLIDE_H = 960, CANVAS_H + CAM_W = CANVAS_W - SLIDE_W # 320 logging.info(f'Compositing {len(slide_events)} slide changes onto video...') @@ -734,20 +748,23 @@ def _composite_slides( # Step 2: Combine slide track + webcam into final layout. # Webcam is input 0 (25fps, drives output frame rate). # Slide track is input 1 (1fps, overlaid on left). + # Determine webcam height: scale to CAM_W, keep aspect ratio, + # but cap at CANVAS_H so it doesn't overflow. _detect_gpu() if _CUDA_OVERLAY_AVAILABLE: filter_graph = ( - f'[0:v]hwupload_cuda,scale_cuda={CAM_W}:{CAM_H}[webcam_gpu];' + f'[0:v]hwupload_cuda,scale_cuda={CAM_W}:-2[webcam_gpu];' f'[1:v]hwupload_cuda,scale_cuda={CANVAS_W}:{CANVAS_H}[slide_gpu];' f'[slide_gpu][webcam_gpu]overlay_cuda={SLIDE_W}:0[out]' ) else: filter_graph = ( - f'[0:v]scale={CAM_W}:{CAM_H}:force_original_aspect_ratio=decrease,' - f'pad={CANVAS_W}:{CANVAS_H}:{SLIDE_W}:0:black[base];' - f'[1:v]scale={SLIDE_W}:{SLIDE_H}:force_original_aspect_ratio=decrease,' - f'pad={SLIDE_W}:{SLIDE_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' - f'[base][slide]overlay=0:0[out]' + f'[0:v]scale={CAM_W}:-2,setsar=1[webcam];' + f'[1:v]scale={SLIDE_W}:{CANVAS_H}:force_original_aspect_ratio=decrease,' + f'pad={SLIDE_W}:{CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' + f'color=c=black:s={CANVAS_W}x{CANVAS_H}:r=25[bg];' + f'[bg][slide]overlay=0:0[tmp];' + f'[tmp][webcam]overlay={SLIDE_W}:0[out]' ) cmd = [ @@ -860,6 +877,14 @@ def compile_final_video( current_time = 0.0 for i, (vpath, start_time) in enumerate(video_files): + # Skip segments whose start_time is before current_time (overlap) + if start_time < current_time - 0.5: + logging.warning( + f'Segment {i} starts at {start_time:.1f}s but current_time ' + f'is {current_time:.1f}s — skipping overlapping segment' + ) + continue + # Insert black gap if needed gap = start_time - current_time if gap > 0.1: # skip tiny gaps < 100ms @@ -868,9 +893,22 @@ def compile_final_video( concat_segments.append(gap_path) logging.info(f'Generated {gap:.1f}s black gap before segment {i}') - # Normalize the segment + # Compute max allowed duration: truncate if this segment would + # overlap the next segment's start_time + max_dur = 0 # 0 = no limit + next_start = None + for j in range(i + 1, len(video_files)): + ns = video_files[j][1] + if ns > start_time + 0.5: + next_start = ns + break + if next_start is not None: + max_dur = next_start - start_time + + # Normalize the segment (with optional truncation) norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') - _normalize_segment(vpath, norm_path, target_w, target_h, target_pix_fmt) + _normalize_segment(vpath, norm_path, target_w, target_h, target_pix_fmt, + max_duration=max_dur) # Ensure it has an audio stream with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') final_seg = _ensure_audio_stream(norm_path, with_audio_path) From c06005e4fff0eeb661cc3060aa46d9a0b0d3e50d Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sat, 4 Apr 2026 09:40:00 +0000 Subject: [PATCH 18/65] Fix grid layout crash and silent audio in merged videos Grid fix: cell dimensions from integer division could be odd, causing ffmpeg's scale filter to round up and produce dimensions larger than the pad target ("Padded dimensions cannot be smaller than input"). Now forces even dimensions and uses min() to cap scale output. Audio fix: amix divides volume by number of inputs at each stage. After 3 levels of mixing (batch->reduce->overlay), audio was attenuated to near-silence (-91 dB). Added volume=N compensation after each amix to restore original loudness. --- mtslinker/processor.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index bda1740..f7d816e 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -238,6 +238,11 @@ def _compute_grid(n: int) -> Tuple[int, int]: return cols, rows +def _even(x: int) -> int: + """Round down to nearest even number (ffmpeg requires even dimensions).""" + return x & ~1 + + def _composite_grid( active_segments: list, duration: float, @@ -255,8 +260,8 @@ def _composite_grid( """ n = len(active_segments) cols, rows = _compute_grid(n) - cell_w = target_w // cols - cell_h = target_h // rows + cell_w = _even(target_w // cols) + cell_h = _even(target_h // rows) inputs = [] filter_parts = [] @@ -265,8 +270,10 @@ def _composite_grid( for i, (path, offset) in enumerate(active_segments): inputs.extend(['-ss', str(offset), '-i', path]) label = f'v{i}' + # Scale to fit cell, then force exact cell dimensions with pad filter_parts.append( - f'[{i}:v]scale={cell_w}:{cell_h}:force_original_aspect_ratio=decrease,' + f'[{i}:v]scale=w=min({cell_w}\\,iw):h=min({cell_h}\\,ih)' + f':force_original_aspect_ratio=decrease,' f'pad={cell_w}:{cell_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[{label}]' ) labels.append(f'[{label}]') @@ -274,13 +281,10 @@ def _composite_grid( # Pad with black cells if needed total_cells = cols * rows for i in range(n, total_cells): + idx = len(active_segments) + i - n inputs.extend(['-f', 'lavfi', '-i', f'color=black:s={cell_w}x{cell_h}:d={duration}:r=25']) - label = f'v{i}' - labels.append(f'[{len(active_segments) + i - n}:v]') - # lavfi inputs don't need scaling, rename - idx = len(active_segments) + i - n - labels[-1] = f'[{idx}:v]' + labels.append(f'[{idx}:v]') # Build xstack layout string: x_y positions layout_parts = [] @@ -1025,10 +1029,13 @@ def _merge_audio_tracks( # Single track — just delay and encode, no amix needed filter_graph = filter_parts[0].rsplit('[', 1)[0] # strip label else: + # amix divides volume by number of inputs — compensate with + # volume filter so the mix stays at original loudness filter_graph = ( ';'.join(filter_parts) + ';' + ''.join(mix_labels) - + f'amix=inputs={len(batch)}:duration=longest:normalize=0' + + f'amix=inputs={len(batch)}:duration=longest:normalize=0,' + + f'volume={len(batch)}' ) try: @@ -1084,7 +1091,8 @@ def _merge_audio_tracks( labels = ''.join(f'[{j}:a]' for j in range(len(batch))) amix_filter = ( f'{labels}amix=inputs={len(batch)}' - f':duration=longest:normalize=0' + f':duration=longest:normalize=0,' + f'volume={len(batch)}' ) _run_ffmpeg( @@ -1124,7 +1132,7 @@ def _merge_audio_tracks( '-i', video_path, '-i', mixed_audio_path, '-filter_complex', - '[0:a][1:a]amix=inputs=2:duration=first:normalize=0[aout]', + '[0:a][1:a]amix=inputs=2:duration=first:normalize=0,volume=2[aout]', '-map', '0:v', '-map', '[aout]', '-c:v', 'copy', '-c:a', 'aac', '-b:a', '192k', From 2f8e54c990e1898edd5ef458ffd3895217aa9133 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sat, 4 Apr 2026 16:49:51 +0000 Subject: [PATCH 19/65] Fix audio: remove volume over-amplification, filter silent segments normalize=0 already prevents amix from dividing by N, so the volume=N multiplier was over-amplifying (~x112 across 3 pipeline stages), turning noise from silent tracks into interference. Also filter out silent audio-only segments (<-80 dB) before mixing so they don't waste processing time or add noise floor. --- mtslinker/processor.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index f7d816e..8fdcd8d 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -111,6 +111,23 @@ def _ffprobe_streams(file_path: str) -> dict: return json.loads(result.stdout) +def _is_silent(file_path: str, threshold: float = -80.0) -> bool: + """Check if an audio file is effectively silent (mean volume below threshold).""" + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-i', file_path, + '-t', '10', '-af', 'volumedetect', '-f', 'null', '-'], + capture_output=True, text=True, + ) + for line in result.stderr.splitlines(): + if 'mean_volume' in line: + try: + vol = float(line.split('mean_volume:')[1].strip().split()[0]) + return vol < threshold + except (ValueError, IndexError): + pass + return True # if we can't detect, treat as silent + + def _has_video_stream(file_path: str) -> bool: """Check if a file contains a video stream.""" info = _ffprobe_streams(file_path) @@ -960,7 +977,15 @@ def compile_final_video( ) video_only_path = composited_path - # If there are audio-only tracks, overlay them + # If there are audio-only tracks, overlay them (skip silent ones) + if audio_files: + orig_count = len(audio_files) + audio_files = [(p, t) for p, t in audio_files if not _is_silent(p)] + if orig_count != len(audio_files): + logging.info( + f'Filtered {orig_count - len(audio_files)}/{orig_count} ' + f'silent audio-only segments' + ) if audio_files: logging.info(f'Merging {len(audio_files)} audio-only tracks...') result_path = _merge_audio_tracks( @@ -1029,13 +1054,11 @@ def _merge_audio_tracks( # Single track — just delay and encode, no amix needed filter_graph = filter_parts[0].rsplit('[', 1)[0] # strip label else: - # amix divides volume by number of inputs — compensate with - # volume filter so the mix stays at original loudness + # normalize=0 prevents amix from dividing by N filter_graph = ( ';'.join(filter_parts) + ';' + ''.join(mix_labels) - + f'amix=inputs={len(batch)}:duration=longest:normalize=0,' - + f'volume={len(batch)}' + + f'amix=inputs={len(batch)}:duration=longest:normalize=0' ) try: @@ -1091,8 +1114,7 @@ def _merge_audio_tracks( labels = ''.join(f'[{j}:a]' for j in range(len(batch))) amix_filter = ( f'{labels}amix=inputs={len(batch)}' - f':duration=longest:normalize=0,' - f'volume={len(batch)}' + f':duration=longest:normalize=0' ) _run_ffmpeg( @@ -1132,7 +1154,7 @@ def _merge_audio_tracks( '-i', video_path, '-i', mixed_audio_path, '-filter_complex', - '[0:a][1:a]amix=inputs=2:duration=first:normalize=0,volume=2[aout]', + '[0:a][1:a]amix=inputs=2:duration=first:normalize=0[aout]', '-map', '0:v', '-map', '[aout]', '-c:v', 'copy', '-c:a', 'aac', '-b:a', '192k', From 24b1bf45dffe11d901d8f12c0cd5320fc656d7a7 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sat, 4 Apr 2026 21:20:58 +0000 Subject: [PATCH 20/65] Lower silent audio threshold from -80 to -88 dB -80 dB was filtering out participant microphone audio that sits around -80 to -60 dB. Only -91 dB is true digital silence. --- mtslinker/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 8fdcd8d..e6fcb8f 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -111,7 +111,7 @@ def _ffprobe_streams(file_path: str) -> dict: return json.loads(result.stdout) -def _is_silent(file_path: str, threshold: float = -80.0) -> bool: +def _is_silent(file_path: str, threshold: float = -88.0) -> bool: """Check if an audio file is effectively silent (mean volume below threshold).""" result = subprocess.run( ['ffmpeg', '-v', 'error', '-i', file_path, From 49480e5dc3f83abde980944363a5f7ecd68232bd Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sat, 4 Apr 2026 23:12:18 +0000 Subject: [PATCH 21/65] =?UTF-8?q?Remove=20silent=20audio=20filter=20?= =?UTF-8?q?=E2=80=94=20unnecessary=20without=20volume=20amplification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With normalize=0 and no volume=N, mixing silent segments with real audio just gives real audio. The filter was incorrectly dropping participant microphone tracks. Removing it simplifies the pipeline and ensures all audio-only segments are included. --- mtslinker/processor.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index e6fcb8f..f03e416 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -977,15 +977,7 @@ def compile_final_video( ) video_only_path = composited_path - # If there are audio-only tracks, overlay them (skip silent ones) - if audio_files: - orig_count = len(audio_files) - audio_files = [(p, t) for p, t in audio_files if not _is_silent(p)] - if orig_count != len(audio_files): - logging.info( - f'Filtered {orig_count - len(audio_files)}/{orig_count} ' - f'silent audio-only segments' - ) + # If there are audio-only tracks, overlay them if audio_files: logging.info(f'Merging {len(audio_files)} audio-only tracks...') result_path = _merge_audio_tracks( From e483fe0f1730d105a19d6151213f207a593f473b Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sat, 4 Apr 2026 23:24:11 +0000 Subject: [PATCH 22/65] Add active speaker switching for slide presentations When slides + multiple webcams are present, analyzes audio levels per participant to detect who is talking. Switches the right-side webcam to show the active speaker, defaulting to presenter when nobody else talks. Uses 2s analysis windows with 4s minimum hold to prevent flickering. --- mtslinker/processor.py | 271 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 268 insertions(+), 3 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index f03e416..25caa91 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -128,6 +128,263 @@ def _is_silent(file_path: str, threshold: float = -88.0) -> bool: return True # if we can't detect, treat as silent +def _analyze_audio_levels(file_path: str, window_sec: float = 2.0, + sample_rate: int = 44100) -> List[Tuple[float, float]]: + """Analyze RMS audio levels per time window. + + Returns [(time_offset_in_file, rms_db), ...] for each window. + """ + import tempfile + reset_samples = int(sample_rate * window_sec) + with tempfile.NamedTemporaryFile(suffix='.txt', delete=False) as tmp: + tmp_path = tmp.name + + try: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-i', file_path, + '-af', f'astats=metadata=1:reset={reset_samples},' + f'ametadata=print:key=lavfi.astats.Overall.RMS_level' + f':file={tmp_path}', + '-f', 'null', '-'], + capture_output=True, text=True, + ) + if result.returncode != 0: + return [] + + levels = [] + current_time = None + with open(tmp_path) as f: + for line in f: + line = line.strip() + if line.startswith('frame:'): + # Extract pts_time from "frame:N pts:N pts_time:N.NNN" + for part in line.split(): + if part.startswith('pts_time:'): + try: + current_time = float(part.split(':')[1]) + except (ValueError, IndexError): + pass + elif 'RMS_level' in line and current_time is not None: + try: + val = float(line.split('=')[1]) + levels.append((current_time, val)) + except (ValueError, IndexError): + pass + return levels + finally: + try: + os.remove(tmp_path) + except OSError: + pass + + +def _build_speaker_timeline( + video_files: list, + total_duration: float, + window_sec: float = 2.0, + silence_threshold: float = -50.0, + min_hold_sec: float = 4.0, +) -> List[Tuple[float, float, str]]: + """Build a timeline of which conf_id should be shown at each moment. + + Picks the loudest non-admin speaker when above silence_threshold, + otherwise defaults to admin. Applies hysteresis to prevent flickering. + + Returns [(interval_start, interval_end, conf_id), ...]. + """ + from collections import defaultdict + + if not video_files: + return [] + + # Group by conf_id, identify admin conf + conf_segments = defaultdict(list) # conf_id -> [(path, start_time)] + admin_conf = None + conf_is_admin = {} + + for item in video_files: + path, start = item[0], item[1] + conf_id = item[2] if len(item) > 2 else None + is_admin = item[3] if len(item) > 3 else False + if conf_id is None: + continue + conf_segments[conf_id].append((path, start)) + if is_admin: + conf_is_admin[conf_id] = True + + if not conf_segments: + return [] + + # Find admin conf (most duration among admin confs, or most duration overall) + conf_total_dur = {} + for conf_id, segs in conf_segments.items(): + conf_total_dur[conf_id] = sum(_get_duration(p) for p, _ in segs) + + admin_confs = {c for c in conf_segments if conf_is_admin.get(c)} + if admin_confs: + admin_conf = max(admin_confs, key=lambda c: conf_total_dur.get(c, 0)) + elif conf_total_dur: + admin_conf = max(conf_total_dur, key=conf_total_dur.get) + + # Analyze audio levels for each segment and map to absolute timeline + # conf_id -> {window_index: max_rms} + n_windows = int(total_duration / window_sec) + 1 + conf_levels = defaultdict(lambda: defaultdict(lambda: -91.0)) + + logging.info(f'Speaker detection: analyzing audio for {len(conf_segments)} participants...') + for conf_id, segs in conf_segments.items(): + for path, seg_start in segs: + levels = _analyze_audio_levels(path, window_sec) + for time_offset, rms in levels: + abs_time = seg_start + time_offset + win_idx = int(abs_time / window_sec) + if 0 <= win_idx < n_windows: + # Keep the max RMS for this conf in this window + if rms > conf_levels[conf_id][win_idx]: + conf_levels[conf_id][win_idx] = rms + + # For each window, pick the speaker + all_confs = list(conf_segments.keys()) + non_admin = [c for c in all_confs if c != admin_conf] + raw_picks = [] + + for win_idx in range(n_windows): + best_non_admin = None + best_rms = silence_threshold + for conf_id in non_admin: + rms = conf_levels[conf_id][win_idx] + if rms > best_rms: + best_rms = rms + best_non_admin = conf_id + raw_picks.append(best_non_admin if best_non_admin else admin_conf) + + # Apply hysteresis: revert short non-admin bursts to admin + min_hold_windows = max(1, int(min_hold_sec / window_sec)) + picks = list(raw_picks) + i = 0 + while i < len(picks): + if picks[i] != admin_conf: + # Find run length of this non-admin speaker + j = i + while j < len(picks) and picks[j] == picks[i]: + j += 1 + if j - i < min_hold_windows: + # Too short, revert to admin + for k in range(i, j): + picks[k] = admin_conf + i = j + else: + i += 1 + + # Merge consecutive same-speaker windows into intervals + intervals = [] + if picks: + current_conf = picks[0] + interval_start = 0.0 + for win_idx in range(1, len(picks)): + if picks[win_idx] != current_conf: + interval_end = win_idx * window_sec + intervals.append((interval_start, interval_end, current_conf)) + current_conf = picks[win_idx] + interval_start = interval_end + # Final interval + intervals.append((interval_start, total_duration, current_conf)) + + # Log summary + non_admin_time = sum( + end - start for start, end, conf in intervals if conf != admin_conf + ) + logging.info( + f'Speaker timeline: {len(intervals)} intervals, ' + f'{non_admin_time:.0f}s non-admin out of {total_duration:.0f}s' + ) + + return intervals + + +def _build_speaker_switched_segments( + video_files: list, + speaker_timeline: List[Tuple[float, float, str]], + tmp_dir: str, + target_w: int, + target_h: int, +) -> List[Tuple[str, float]]: + """Cut webcam segments according to the speaker timeline. + + For each interval, extracts the appropriate slice from the source + webcam segment for that conf_id. Returns [(path, start_time), ...] + in the same format as _deduplicate_overlapping. + """ + if not speaker_timeline: + return [] + + # Build lookup: conf_id -> sorted [(path, start, duration, end)] + from collections import defaultdict + conf_segs = defaultdict(list) + for item in video_files: + path, start = item[0], item[1] + conf_id = item[2] if len(item) > 2 else None + dur = _get_duration(path) + conf_segs[conf_id].append((path, start, dur, start + dur)) + for conf_id in conf_segs: + conf_segs[conf_id].sort(key=lambda x: x[1]) + + speaker_dir = os.path.join(tmp_dir, 'speaker_segments') + os.makedirs(speaker_dir, exist_ok=True) + + result = [] + for i, (t_start, t_end, conf_id) in enumerate(speaker_timeline): + duration = t_end - t_start + if duration < 0.1: + continue + + # Find the source segment covering this interval + source = None + for path, seg_start, seg_dur, seg_end in conf_segs.get(conf_id, []): + if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: + source = (path, seg_start) + break + + if source is None: + # No segment for this conf_id at this time — try any conf + for cid, segs in conf_segs.items(): + for path, seg_start, seg_dur, seg_end in segs: + if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: + source = (path, seg_start) + break + if source: + break + + if source is None: + continue + + src_path, seg_start = source + offset = t_start - seg_start + seg_path = os.path.join(speaker_dir, f'speaker_{i:04d}.mp4') + + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, offset)), + '-i', src_path, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, conf {conf_id})', + ) + result.append((seg_path, t_start)) + + logging.info(f'Speaker switching: built {len(result)} segments') + return result + + def _has_video_stream(file_path: str) -> bool: """Check if a file contains a video stream.""" info = _ffprobe_streams(file_path) @@ -883,16 +1140,24 @@ def compile_final_video( has_overlaps = True break - if has_overlaps and not slide_events: + if has_overlaps and slide_events: + # Active speaker switching: show the talking person's webcam + logging.info('Multiple concurrent webcams + slides, using speaker switching') + speaker_timeline = _build_speaker_timeline( + video_files, total_duration, + ) + video_files = _build_speaker_switched_segments( + video_files, speaker_timeline, tmp_dir, target_w, target_h, + ) + elif has_overlaps: # Grid layout: composite all concurrent webcams into a grid logging.info('Multiple concurrent webcams detected, building grid layout') video_files = _build_grid_segments( video_files, tmp_dir, target_w, target_h, total_duration, ) else: - # Deduplicate overlapping segments (keep presenter for slide videos) video_files = _deduplicate_overlapping(video_files) - logging.info(f'After dedup/grid: {len(video_files)} segments') + logging.info(f'After dedup/grid/speaker: {len(video_files)} segments') concat_segments = [] current_time = 0.0 From e614b8340302b7b4a8243f818db9104b591c1c56 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sun, 5 Apr 2026 11:45:33 +0000 Subject: [PATCH 23/65] Cap target resolution at 720p to prevent OOM kills on long lectures --- mtslinker/processor.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 25caa91..7caa0db 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1119,6 +1119,12 @@ def compile_final_video( # Minimum 640x360 for reasonable quality if target_w * target_h < 640 * 360: target_w, target_h = 640, 360 + # Cap at 720p to avoid OOM on long lectures with many segments + MAX_H = 720 + if target_h > MAX_H: + target_w = int(target_w * MAX_H / target_h) + target_w -= target_w % 2 # keep even + target_h = MAX_H logging.info(f'Target resolution: {target_w}x{target_h}, pix_fmt={target_pix_fmt}') # Sort by start time From dd04fb478a4b82c7275f56e40283f302434966aa Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sun, 5 Apr 2026 15:53:50 +0000 Subject: [PATCH 24/65] Use libx264 for slide compositing to avoid OOM on 8GB RAM NVENC + complex overlay filter on 3+ hour videos consumes ~7GB, triggering OOM killer. libx264 uses ~300MB for the same operation. All other encoding steps still use NVENC. --- mtslinker/processor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 7caa0db..7d50bc1 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1051,7 +1051,8 @@ def _composite_slides( '-i', slide_track_path, '-filter_complex', filter_graph, '-map', '[out]', '-map', '0:a?', - *_get_video_encoder(), + # Force libx264 for compositing — NVENC + long overlay OOMs on 8GB RAM + '-c:v', 'libx264', '-preset', 'fast', '-crf', '23', '-c:a', 'copy', '-r', '25', '-shortest', From bd4647b08a49906929bb47d4966fefae514243cd Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sun, 5 Apr 2026 21:20:37 +0000 Subject: [PATCH 25/65] Lower resolution cap to 480p and use ultrafast preset to fix OOM The 720p cap + fast preset still OOM-kills on 3.5h recordings with many segments (e.g. 1197678196: 125 chunks, 23 participants, 34 segments). --- mtslinker/processor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 7d50bc1..0cac54d 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1052,7 +1052,7 @@ def _composite_slides( '-filter_complex', filter_graph, '-map', '[out]', '-map', '0:a?', # Force libx264 for compositing — NVENC + long overlay OOMs on 8GB RAM - '-c:v', 'libx264', '-preset', 'fast', '-crf', '23', + '-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23', '-c:a', 'copy', '-r', '25', '-shortest', @@ -1120,8 +1120,8 @@ def compile_final_video( # Minimum 640x360 for reasonable quality if target_w * target_h < 640 * 360: target_w, target_h = 640, 360 - # Cap at 720p to avoid OOM on long lectures with many segments - MAX_H = 720 + # Cap at 480p to avoid OOM on long lectures with many segments + MAX_H = 480 if target_h > MAX_H: target_w = int(target_w * MAX_H / target_h) target_w -= target_w % 2 # keep even From 05a3255eff9cc28f2f1633ae1a393cbc72845219 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 6 Apr 2026 12:23:19 +0000 Subject: [PATCH 26/65] Chunked slide compositing to prevent OOM, restore GPU encoding Split compositing into 30-min chunks so ffmpeg never holds the full video in memory. This allows using NVENC again (faster) and restores 720p resolution cap. Each chunk is composited independently then concatenated with stream copy. --- mtslinker/processor.py | 86 ++++++++++++++++++++++++++++++++---------- 1 file changed, 66 insertions(+), 20 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 0cac54d..3d79341 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1024,10 +1024,9 @@ def _composite_slides( logging.info('Slide track created') # Step 2: Combine slide track + webcam into final layout. - # Webcam is input 0 (25fps, drives output frame rate). - # Slide track is input 1 (1fps, overlaid on left). - # Determine webcam height: scale to CAM_W, keep aspect ratio, - # but cap at CANVAS_H so it doesn't overflow. + # Process in chunks to avoid OOM on long videos. + CHUNK_SECS = 1800 # 30 minutes per chunk + _detect_gpu() if _CUDA_OVERLAY_AVAILABLE: filter_graph = ( @@ -1045,21 +1044,68 @@ def _composite_slides( f'[tmp][webcam]overlay={SLIDE_W}:0[out]' ) - cmd = [ - 'ffmpeg', '-y', '-v', 'warning', - '-i', video_path, - '-i', slide_track_path, - '-filter_complex', filter_graph, - '-map', '[out]', '-map', '0:a?', - # Force libx264 for compositing — NVENC + long overlay OOMs on 8GB RAM - '-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23', - '-c:a', 'copy', - '-r', '25', - '-shortest', - output_path, - ] + n_chunks = max(1, int(total_duration + CHUNK_SECS - 1) // CHUNK_SECS) + + enc_args = _get_video_encoder() + + if n_chunks == 1: + # Short video — composite in one pass + cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', video_path, + '-i', slide_track_path, + '-filter_complex', filter_graph, + '-map', '[out]', '-map', '0:a?', + *enc_args, + '-c:a', 'copy', + '-r', '25', + '-shortest', + output_path, + ] + _run_ffmpeg(cmd, description='composite slides + webcam') + else: + # Long video — split into chunks, composite each, then concat + logging.info(f'Compositing in {n_chunks} chunks of {CHUNK_SECS}s to avoid OOM') + chunks_dir = os.path.join(tmp_dir, 'composite_chunks') + os.makedirs(chunks_dir, exist_ok=True) + chunk_paths = [] + + for ci in range(n_chunks): + ss = ci * CHUNK_SECS + chunk_path = os.path.join(chunks_dir, f'chunk_{ci:03d}.mp4') + cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-ss', str(ss), '-t', str(CHUNK_SECS), + '-i', video_path, + '-ss', str(ss), '-t', str(CHUNK_SECS), + '-i', slide_track_path, + '-filter_complex', filter_graph, + '-map', '[out]', '-map', '0:a?', + *enc_args, + '-c:a', 'aac', '-b:a', '192k', + '-r', '25', + '-shortest', + chunk_path, + ] + _run_ffmpeg(cmd, description=f'composite chunk {ci+1}/{n_chunks}') + chunk_paths.append(chunk_path) + logging.info(f'Composite chunk {ci+1}/{n_chunks} done') + + # Concatenate chunks + chunk_list = os.path.join(chunks_dir, 'concat.txt') + with open(chunk_list, 'w') as f: + for cp in chunk_paths: + f.write(f"file '{os.path.abspath(cp)}'\n") + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-f', 'concat', '-safe', '0', '-i', chunk_list, + '-c', 'copy', output_path, + ], + description='concat composite chunks', + ) + shutil.rmtree(chunks_dir, ignore_errors=True) - _run_ffmpeg(cmd, description='composite slides + webcam') logging.info('Slide compositing complete') # Cleanup slide segments @@ -1120,8 +1166,8 @@ def compile_final_video( # Minimum 640x360 for reasonable quality if target_w * target_h < 640 * 360: target_w, target_h = 640, 360 - # Cap at 480p to avoid OOM on long lectures with many segments - MAX_H = 480 + # Cap at 720p to avoid excessive memory use + MAX_H = 720 if target_h > MAX_H: target_w = int(target_w * MAX_H / target_h) target_w -= target_w % 2 # keep even From 8bbfe962eac030243dc72a212ab7bfb1ac8f1b1d Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 6 Apr 2026 14:09:13 +0000 Subject: [PATCH 27/65] Fix CUDA overlay detection skipped when encoder_fast called first _get_video_encoder_fast() set _NVENC_AVAILABLE directly, bypassing _detect_gpu(). This left _CUDA_OVERLAY_AVAILABLE as None, so compositing always used CPU overlay even with CUDA support available. --- mtslinker/processor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 3d79341..43a8d14 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -63,9 +63,7 @@ def _get_video_encoder() -> list: def _get_video_encoder_fast() -> list: """Return fast encoder args for simple content (slides, gaps).""" - global _NVENC_AVAILABLE - if _NVENC_AVAILABLE is None: - _NVENC_AVAILABLE = _has_nvenc() + _detect_gpu() if _NVENC_AVAILABLE: return ['-c:v', 'h264_nvenc', '-preset', 'p1', '-cq', '28'] return ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23'] From 75b4457e1adf2f05a63d7586a7869e57171c09ec Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 6 Apr 2026 14:14:56 +0000 Subject: [PATCH 28/65] Parallelize speaker detection audio analysis with 4 threads Each participant's audio segments are analyzed by independent ffmpeg calls. Running 4 in parallel instead of sequentially speeds up speaker detection ~4x on multi-core systems. --- mtslinker/processor.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 43a8d14..7fa5c0f 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -4,6 +4,7 @@ import os import shutil import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Dict, List, Tuple, Union AUDIO_MERGE_BATCH_SIZE = 8 @@ -230,14 +231,26 @@ def _build_speaker_timeline( conf_levels = defaultdict(lambda: defaultdict(lambda: -91.0)) logging.info(f'Speaker detection: analyzing audio for {len(conf_segments)} participants...') + + # Build list of (conf_id, path, seg_start) tasks for parallel analysis + analysis_tasks = [] for conf_id, segs in conf_segments.items(): for path, seg_start in segs: - levels = _analyze_audio_levels(path, window_sec) + analysis_tasks.append((conf_id, path, seg_start)) + + def _analyze_task(task): + conf_id, path, seg_start = task + levels = _analyze_audio_levels(path, window_sec) + return conf_id, seg_start, levels + + with ThreadPoolExecutor(max_workers=4) as pool: + futures = {pool.submit(_analyze_task, t): t for t in analysis_tasks} + for fut in as_completed(futures): + conf_id, seg_start, levels = fut.result() for time_offset, rms in levels: abs_time = seg_start + time_offset win_idx = int(abs_time / window_sec) if 0 <= win_idx < n_windows: - # Keep the max RMS for this conf in this window if rms > conf_levels[conf_id][win_idx]: conf_levels[conf_id][win_idx] = rms From e7202793e21396d7c1f7114fd9c1276a3230a45f Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 6 Apr 2026 17:33:06 +0000 Subject: [PATCH 29/65] Cap concat output at total_duration to prevent inflated video length Speaker switching can produce segments whose combined duration exceeds the original recording. Cap the concat at total_duration from the API to ensure the output matches the expected length. --- mtslinker/processor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 7fa5c0f..cfe8c3f 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1291,8 +1291,10 @@ def compile_final_video( '-c', 'copy', ] - if max_duration: - concat_cmd.extend(['-t', str(max_duration)]) + # Cap at max_duration or total_duration to prevent inflated output + cap = max_duration or total_duration + if cap: + concat_cmd.extend(['-t', str(cap)]) concat_cmd.append(video_only_path) _run_ffmpeg(concat_cmd, description=f'concat {len(concat_segments)} segments') From 682c617c06231f13049c32cdf634fb124bee8eca Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 6 Apr 2026 18:13:46 +0000 Subject: [PATCH 30/65] Fix duration inflation: track intended timeline instead of file duration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit compile_final_video used _get_duration(file) to advance current_time. Re-encoded segments drift slightly longer, compounding over many segments (34 segments → 2.6x inflation on a 3.5h recording). Now uses the intended timeline duration (next_start - start_time) to advance current_time, keeping the output aligned with the API's total_duration. --- mtslinker/processor.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index cfe8c3f..e87ec2c 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1264,8 +1264,16 @@ def compile_final_video( final_seg = _ensure_audio_stream(norm_path, with_audio_path) concat_segments.append(final_seg) - seg_dur = _get_duration(final_seg) - current_time = start_time + seg_dur + # Use intended duration (from timeline) to advance current_time, + # not actual file duration which may drift due to re-encoding. + if max_dur > 0: + current_time = start_time + max_dur + elif next_start is not None: + current_time = next_start + else: + # Last segment — use actual duration as fallback + seg_dur = _get_duration(final_seg) + current_time = start_time + seg_dur # Trailing gap if current_time < total_duration - 0.1: From 16849da26dc0212a64382eacfdbb4d3fb169ec53 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 6 Apr 2026 20:39:40 +0000 Subject: [PATCH 31/65] Fix slide compositing: correct layout and force 25fps CUDA overlay path had webcam covering slides (wrong overlay order) and output was 1fps (inherited from slide track). Both GPU and CPU paths now use the same correct filter graph with fps=25 on both inputs: slide left (960px), webcam top-right (320px), black below. --- mtslinker/processor.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index e87ec2c..2c30795 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1040,15 +1040,21 @@ def _composite_slides( _detect_gpu() if _CUDA_OVERLAY_AVAILABLE: + # Build canvas on CPU (slide left, webcam top-right on black bg) + # then upload to CUDA for encoding. overlay_cuda has limited layout + # control, so we use CPU overlay for correct positioning. filter_graph = ( - f'[0:v]hwupload_cuda,scale_cuda={CAM_W}:-2[webcam_gpu];' - f'[1:v]hwupload_cuda,scale_cuda={CANVAS_W}:{CANVAS_H}[slide_gpu];' - f'[slide_gpu][webcam_gpu]overlay_cuda={SLIDE_W}:0[out]' + f'[0:v]fps=25,scale={CAM_W}:-2,setsar=1[webcam];' + f'[1:v]fps=25,scale={SLIDE_W}:{CANVAS_H}:force_original_aspect_ratio=decrease,' + f'pad={SLIDE_W}:{CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' + f'color=c=black:s={CANVAS_W}x{CANVAS_H}:r=25[bg];' + f'[bg][slide]overlay=0:0[tmp];' + f'[tmp][webcam]overlay={SLIDE_W}:0[out]' ) else: filter_graph = ( - f'[0:v]scale={CAM_W}:-2,setsar=1[webcam];' - f'[1:v]scale={SLIDE_W}:{CANVAS_H}:force_original_aspect_ratio=decrease,' + f'[0:v]fps=25,scale={CAM_W}:-2,setsar=1[webcam];' + f'[1:v]fps=25,scale={SLIDE_W}:{CANVAS_H}:force_original_aspect_ratio=decrease,' f'pad={SLIDE_W}:{CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' f'color=c=black:s={CANVAS_W}x{CANVAS_H}:r=25[bg];' f'[bg][slide]overlay=0:0[tmp];' From b4ce2f17551dbff08a79a074e35bcfcb3fafc246 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 7 Apr 2026 12:37:40 +0000 Subject: [PATCH 32/65] Mix admin audio into non-admin speaker segments When showing a participant's webcam during speaker switching, the organizer's voice was lost because only the participant's audio was included. Now mixes the admin's webcam audio into non-admin segments so both voices are always audible. Uses the proper admin_conf from API is_admin flags, passed from _build_speaker_timeline to _build_speaker_switched_segments. --- mtslinker/processor.py | 103 +++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 34 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 2c30795..0f08361 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -310,12 +310,13 @@ def _analyze_task(task): f'{non_admin_time:.0f}s non-admin out of {total_duration:.0f}s' ) - return intervals + return intervals, admin_conf def _build_speaker_switched_segments( video_files: list, speaker_timeline: List[Tuple[float, float, str]], + admin_conf: str, tmp_dir: str, target_w: int, target_h: int, @@ -343,53 +344,86 @@ def _build_speaker_switched_segments( speaker_dir = os.path.join(tmp_dir, 'speaker_segments') os.makedirs(speaker_dir, exist_ok=True) + def _find_source(conf_id, t_start): + """Find the source segment covering t_start for given conf_id.""" + for path, seg_start, seg_dur, seg_end in conf_segs.get(conf_id, []): + if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: + return path, seg_start + return None, None + result = [] for i, (t_start, t_end, conf_id) in enumerate(speaker_timeline): duration = t_end - t_start if duration < 0.1: continue - # Find the source segment covering this interval - source = None - for path, seg_start, seg_dur, seg_end in conf_segs.get(conf_id, []): - if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: - source = (path, seg_start) - break + # Find the source segment for the active speaker + src_path, seg_start = _find_source(conf_id, t_start) - if source is None: - # No segment for this conf_id at this time — try any conf + if src_path is None: + # Fallback: try any conf for cid, segs in conf_segs.items(): - for path, seg_start, seg_dur, seg_end in segs: - if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: - source = (path, seg_start) + for path, ss, sd, se in segs: + if ss <= t_start + 0.5 and se >= t_start + 0.5: + src_path, seg_start = path, ss break - if source: + if src_path: break - if source is None: + if src_path is None: continue - src_path, seg_start = source offset = t_start - seg_start seg_path = os.path.join(speaker_dir, f'speaker_{i:04d}.mp4') - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, offset)), - '-i', src_path, - '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, conf {conf_id})', - ) + # If this is a non-admin segment, mix admin audio in so the + # organizer's voice is always audible (e.g. answering questions) + admin_src, admin_start = None, None + if conf_id != admin_conf and admin_conf is not None: + admin_src, admin_start = _find_source(admin_conf, t_start) + + if admin_src is not None: + admin_offset = t_start - admin_start + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, offset)), + '-i', src_path, + '-ss', str(max(0, admin_offset)), + '-i', admin_src, + '-t', str(duration), + '-filter_complex', + f'[0:v]scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[v];' + f'[0:a][1:a]amix=inputs=2:duration=first:normalize=0[a]', + '-map', '[v]', '-map', '[a]', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, conf {conf_id} + admin audio)', + ) + else: + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, offset)), + '-i', src_path, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, conf {conf_id})', + ) result.append((seg_path, t_start)) logging.info(f'Speaker switching: built {len(result)} segments') @@ -1213,11 +1247,12 @@ def compile_final_video( if has_overlaps and slide_events: # Active speaker switching: show the talking person's webcam logging.info('Multiple concurrent webcams + slides, using speaker switching') - speaker_timeline = _build_speaker_timeline( + speaker_timeline, admin_conf = _build_speaker_timeline( video_files, total_duration, ) video_files = _build_speaker_switched_segments( - video_files, speaker_timeline, tmp_dir, target_w, target_h, + video_files, speaker_timeline, admin_conf, + tmp_dir, target_w, target_h, ) elif has_overlaps: # Grid layout: composite all concurrent webcams into a grid From 97062e737d366346ca4c0b20d80c7e23202fa0d1 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 7 Apr 2026 16:00:42 +0000 Subject: [PATCH 33/65] Always show admin webcam, add participant below when they speak Admin webcam is always visible (top half). When a non-admin speaks, their webcam appears in the bottom half with audio mixed from both. This ensures the organizer's voice and video are never lost during speaker switching. --- mtslinker/processor.py | 153 +++++++++++++++++++++++++++-------------- 1 file changed, 100 insertions(+), 53 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 0f08361..e2eb472 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -323,15 +323,14 @@ def _build_speaker_switched_segments( ) -> List[Tuple[str, float]]: """Cut webcam segments according to the speaker timeline. - For each interval, extracts the appropriate slice from the source - webcam segment for that conf_id. Returns [(path, start_time), ...] - in the same format as _deduplicate_overlapping. + Layout: admin webcam always shown (top). When a non-admin speaks, + their webcam appears below the admin's. Admin audio is always present. + Returns [(path, start_time), ...] for concatenation. """ if not speaker_timeline: return [] # Build lookup: conf_id -> sorted [(path, start, duration, end)] - from collections import defaultdict conf_segs = defaultdict(list) for item in video_files: path, start = item[0], item[1] @@ -344,6 +343,9 @@ def _build_speaker_switched_segments( speaker_dir = os.path.join(tmp_dir, 'speaker_segments') os.makedirs(speaker_dir, exist_ok=True) + half_h = target_h // 2 + half_h -= half_h % 2 # keep even + def _find_source(conf_id, t_start): """Find the source segment covering t_start for given conf_id.""" for path, seg_start, seg_dur, seg_end in conf_segs.get(conf_id, []): @@ -357,56 +359,27 @@ def _find_source(conf_id, t_start): if duration < 0.1: continue - # Find the source segment for the active speaker - src_path, seg_start = _find_source(conf_id, t_start) - - if src_path is None: - # Fallback: try any conf - for cid, segs in conf_segs.items(): - for path, ss, sd, se in segs: - if ss <= t_start + 0.5 and se >= t_start + 0.5: - src_path, seg_start = path, ss - break - if src_path: - break - - if src_path is None: - continue - - offset = t_start - seg_start seg_path = os.path.join(speaker_dir, f'speaker_{i:04d}.mp4') - # If this is a non-admin segment, mix admin audio in so the - # organizer's voice is always audible (e.g. answering questions) - admin_src, admin_start = None, None - if conf_id != admin_conf and admin_conf is not None: - admin_src, admin_start = _find_source(admin_conf, t_start) - - if admin_src is not None: - admin_offset = t_start - admin_start - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, offset)), - '-i', src_path, - '-ss', str(max(0, admin_offset)), - '-i', admin_src, - '-t', str(duration), - '-filter_complex', - f'[0:v]scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[v];' - f'[0:a][1:a]amix=inputs=2:duration=first:normalize=0[a]', - '-map', '[v]', '-map', '[a]', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, conf {conf_id} + admin audio)', - ) - else: + # Always find admin source + admin_src, admin_start = _find_source(admin_conf, t_start) \ + if admin_conf else (None, None) + + if conf_id == admin_conf or admin_conf is None: + # Admin interval: show admin webcam full size + src_path, seg_start = _find_source(conf_id, t_start) + if src_path is None: + # Fallback: try any conf + for cid, segs in conf_segs.items(): + for path, ss, sd, se in segs: + if ss <= t_start + 0.5 and se >= t_start + 0.5: + src_path, seg_start = path, ss + break + if src_path: + break + if src_path is None: + continue + offset = t_start - seg_start _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', @@ -422,8 +395,82 @@ def _find_source(conf_id, t_start): seg_path, ], description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, conf {conf_id})', + f'({duration:.0f}s, admin)', ) + else: + # Non-admin interval: admin top + participant bottom, mixed audio + participant_src, participant_start = _find_source(conf_id, t_start) + if participant_src is None: + # No participant source — just use admin + if admin_src is None: + continue + offset = t_start - admin_start + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, offset)), + '-i', admin_src, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, admin fallback)', + ) + elif admin_src is not None: + # Both available: admin top, participant bottom, mixed audio + p_offset = t_start - participant_start + a_offset = t_start - admin_start + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, a_offset)), + '-i', admin_src, + '-ss', str(max(0, p_offset)), + '-i', participant_src, + '-t', str(duration), + '-filter_complex', + f'[0:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[admin];' + f'[1:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[part];' + f'[admin][part]vstack[v];' + f'[0:a][1:a]amix=inputs=2:duration=first:normalize=0[a]', + '-map', '[v]', '-map', '[a]', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, admin+participant)', + ) + else: + # No admin source — just use participant + p_offset = t_start - participant_start + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, p_offset)), + '-i', participant_src, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, participant only)', + ) result.append((seg_path, t_start)) logging.info(f'Speaker switching: built {len(result)} segments') From 7613e3c712f340894f974d7cfde1c220ae9d63a5 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 7 Apr 2026 16:41:37 +0000 Subject: [PATCH 34/65] Handle vstack failure gracefully in speaker switching When the dual-webcam (admin top + participant bottom) ffmpeg command fails (e.g. source has no frames at seek position), fall back to admin-only instead of crashing the whole pipeline. --- mtslinker/processor.py | 71 +++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index e2eb472..7c4f5bf 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -426,31 +426,52 @@ def _find_source(conf_id, t_start): # Both available: admin top, participant bottom, mixed audio p_offset = t_start - participant_start a_offset = t_start - admin_start - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, a_offset)), - '-i', admin_src, - '-ss', str(max(0, p_offset)), - '-i', participant_src, - '-t', str(duration), - '-filter_complex', - f'[0:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[admin];' - f'[1:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[part];' - f'[admin][part]vstack[v];' - f'[0:a][1:a]amix=inputs=2:duration=first:normalize=0[a]', - '-map', '[v]', '-map', '[a]', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, admin+participant)', - ) + try: + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, a_offset)), + '-i', admin_src, + '-ss', str(max(0, p_offset)), + '-i', participant_src, + '-t', str(duration), + '-filter_complex', + f'[0:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[admin];' + f'[1:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[part];' + f'[admin][part]vstack[v];' + f'[0:a][1:a]amix=inputs=2:duration=first:normalize=0[a]', + '-map', '[v]', '-map', '[a]', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, admin+participant)', + ) + except subprocess.CalledProcessError: + # Fallback: admin only if vstack fails + logging.warning(f'Dual-webcam failed for segment {i}, using admin only') + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, a_offset)), + '-i', admin_src, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, admin fallback after vstack fail)', + ) else: # No admin source — just use participant p_offset = t_start - participant_start From 3336a7c081c72542a44ad27e30701b36b2b9eab9 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 7 Apr 2026 17:20:49 +0000 Subject: [PATCH 35/65] Fix missing defaultdict import and early return types _build_speaker_switched_segments used defaultdict without importing it (NameError crash). Also fix _build_speaker_timeline early returns to return ([], None) tuple matching the new signature. --- mtslinker/processor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 7c4f5bf..d2307b8 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -4,6 +4,7 @@ import os import shutil import subprocess +from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Dict, List, Tuple, Union @@ -194,7 +195,7 @@ def _build_speaker_timeline( from collections import defaultdict if not video_files: - return [] + return [], None # Group by conf_id, identify admin conf conf_segments = defaultdict(list) # conf_id -> [(path, start_time)] @@ -212,7 +213,7 @@ def _build_speaker_timeline( conf_is_admin[conf_id] = True if not conf_segments: - return [] + return [], None # Find admin conf (most duration among admin confs, or most duration overall) conf_total_dur = {} From 32e9379d71ca9da53d7b131b5d8d288339809199 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 7 Apr 2026 19:23:33 +0000 Subject: [PATCH 36/65] Mix ALL webcam audio in speaker segments, not just admin+participant When a non-admin speaks, audio from all other concurrent webcam streams is now mixed in (not just admin + shown participant). This ensures voices from third parties asking questions are heard. --- mtslinker/processor.py | 148 +++++++++++++++++++++++++++++++---------- 1 file changed, 114 insertions(+), 34 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index d2307b8..a3e08ef 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -354,6 +354,18 @@ def _find_source(conf_id, t_start): return path, seg_start return None, None + def _find_all_sources(t_start): + """Find ALL webcam sources covering t_start, returns [(path, offset)].""" + sources = [] + seen = set() + for conf_id, segs in conf_segs.items(): + for path, seg_start, seg_dur, seg_end in segs: + if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: + if path not in seen: + seen.add(path) + sources.append((path, t_start - seg_start)) + return sources + result = [] for i, (t_start, t_end, conf_id) in enumerate(speaker_timeline): duration = t_end - t_start @@ -424,25 +436,54 @@ def _find_source(conf_id, t_start): f'({duration:.0f}s, admin fallback)', ) elif admin_src is not None: - # Both available: admin top, participant bottom, mixed audio + # Admin top + participant bottom, mix ALL available audio p_offset = t_start - participant_start a_offset = t_start - admin_start + + # Find all webcam audio sources at this time + all_sources = _find_all_sources(t_start) + + # Build inputs: 0=admin(video+audio), 1=participant(video+audio), + # 2..N=extra audio sources + inputs = [ + '-ss', str(max(0, a_offset)), '-i', admin_src, + '-ss', str(max(0, p_offset)), '-i', participant_src, + ] + extra_audio = [] + for src_path_extra, src_offset in all_sources: + if src_path_extra in (admin_src, participant_src): + continue + idx = len(inputs) // 4 + len(extra_audio) + 2 + inputs.extend(['-ss', str(max(0, src_offset)), + '-i', src_path_extra]) + extra_audio.append(idx) + + # Build audio mix filter + n_audio = 2 + len(extra_audio) + audio_labels = '[0:a][1:a]' + ''.join( + f'[{idx}:a]' for idx in extra_audio + ) + audio_filter = ( + f'{audio_labels}amix=inputs={n_audio}' + f':duration=first:normalize=0[a]' + ) + + vfilter = ( + f'[0:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[admin];' + f'[1:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[part];' + f'[admin][part]vstack[v];' + f'{audio_filter}' + ) + try: _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, a_offset)), - '-i', admin_src, - '-ss', str(max(0, p_offset)), - '-i', participant_src, + *inputs, '-t', str(duration), - '-filter_complex', - f'[0:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[admin];' - f'[1:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[part];' - f'[admin][part]vstack[v];' - f'[0:a][1:a]amix=inputs=2:duration=first:normalize=0[a]', + '-filter_complex', vfilter, '-map', '[v]', '-map', '[a]', *_get_video_encoder_fast(), '-pix_fmt', 'yuv420p', @@ -451,11 +492,11 @@ def _find_source(conf_id, t_start): seg_path, ], description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, admin+participant)', + f'({duration:.0f}s, admin+participant+{len(extra_audio)} extra)', ) except subprocess.CalledProcessError: - # Fallback: admin only if vstack fails - logging.warning(f'Dual-webcam failed for segment {i}, using admin only') + # Fallback: admin only if complex mix fails + logging.warning(f'Multi-audio failed for segment {i}, using admin only') _run_ffmpeg( [ 'ffmpeg', '-y', '-v', 'error', @@ -471,28 +512,67 @@ def _find_source(conf_id, t_start): seg_path, ], description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, admin fallback after vstack fail)', + f'({duration:.0f}s, admin fallback)', ) else: - # No admin source — just use participant + # No admin source — participant + all other audio p_offset = t_start - participant_start - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, p_offset)), - '-i', participant_src, - '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, participant only)', - ) + all_sources = _find_all_sources(t_start) + inputs = ['-ss', str(max(0, p_offset)), '-i', participant_src] + extra_audio = [] + for src_path_extra, src_offset in all_sources: + if src_path_extra == participant_src: + continue + idx = 1 + len(extra_audio) + inputs.extend(['-ss', str(max(0, src_offset)), + '-i', src_path_extra]) + extra_audio.append(idx) + + if extra_audio: + n_audio = 1 + len(extra_audio) + audio_labels = '[0:a]' + ''.join( + f'[{idx}:a]' for idx in extra_audio + ) + audio_filter = ( + f'{audio_labels}amix=inputs={n_audio}' + f':duration=first:normalize=0[a]' + ) + vf = ( + f'[0:v]scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[v];' + f'{audio_filter}' + ) + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, '-t', str(duration), + '-filter_complex', vf, + '-map', '[v]', '-map', '[a]', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, participant+{len(extra_audio)} extra)', + ) + else: + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, participant only)', + ) result.append((seg_path, t_start)) logging.info(f'Speaker switching: built {len(result)} segments') From b7f4c05a7e93d0b7cf85f4db925467ab44c85fa1 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 7 Apr 2026 22:38:21 +0000 Subject: [PATCH 37/65] Revert speaker switching, use dedup + mix all webcam audio Reverts complex speaker switching audio mixing (caused silent output). Instead: keep admin webcam via dedup, move all other webcam video files to audio_files so their audio gets mixed in the final _merge_audio_tracks step. This ensures all participant voices are captured without complex per-segment audio mixing. --- mtslinker/processor.py | 289 ++++++++--------------------------------- 1 file changed, 53 insertions(+), 236 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index a3e08ef..b45f87e 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -4,7 +4,6 @@ import os import shutil import subprocess -from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Dict, List, Tuple, Union @@ -195,7 +194,7 @@ def _build_speaker_timeline( from collections import defaultdict if not video_files: - return [], None + return [] # Group by conf_id, identify admin conf conf_segments = defaultdict(list) # conf_id -> [(path, start_time)] @@ -213,7 +212,7 @@ def _build_speaker_timeline( conf_is_admin[conf_id] = True if not conf_segments: - return [], None + return [] # Find admin conf (most duration among admin confs, or most duration overall) conf_total_dur = {} @@ -311,27 +310,27 @@ def _analyze_task(task): f'{non_admin_time:.0f}s non-admin out of {total_duration:.0f}s' ) - return intervals, admin_conf + return intervals def _build_speaker_switched_segments( video_files: list, speaker_timeline: List[Tuple[float, float, str]], - admin_conf: str, tmp_dir: str, target_w: int, target_h: int, ) -> List[Tuple[str, float]]: """Cut webcam segments according to the speaker timeline. - Layout: admin webcam always shown (top). When a non-admin speaks, - their webcam appears below the admin's. Admin audio is always present. - Returns [(path, start_time), ...] for concatenation. + For each interval, extracts the appropriate slice from the source + webcam segment for that conf_id. Returns [(path, start_time), ...] + in the same format as _deduplicate_overlapping. """ if not speaker_timeline: return [] # Build lookup: conf_id -> sorted [(path, start, duration, end)] + from collections import defaultdict conf_segs = defaultdict(list) for item in video_files: path, start = item[0], item[1] @@ -344,235 +343,53 @@ def _build_speaker_switched_segments( speaker_dir = os.path.join(tmp_dir, 'speaker_segments') os.makedirs(speaker_dir, exist_ok=True) - half_h = target_h // 2 - half_h -= half_h % 2 # keep even - - def _find_source(conf_id, t_start): - """Find the source segment covering t_start for given conf_id.""" - for path, seg_start, seg_dur, seg_end in conf_segs.get(conf_id, []): - if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: - return path, seg_start - return None, None - - def _find_all_sources(t_start): - """Find ALL webcam sources covering t_start, returns [(path, offset)].""" - sources = [] - seen = set() - for conf_id, segs in conf_segs.items(): - for path, seg_start, seg_dur, seg_end in segs: - if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: - if path not in seen: - seen.add(path) - sources.append((path, t_start - seg_start)) - return sources - result = [] for i, (t_start, t_end, conf_id) in enumerate(speaker_timeline): duration = t_end - t_start if duration < 0.1: continue - seg_path = os.path.join(speaker_dir, f'speaker_{i:04d}.mp4') + # Find the source segment covering this interval + source = None + for path, seg_start, seg_dur, seg_end in conf_segs.get(conf_id, []): + if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: + source = (path, seg_start) + break - # Always find admin source - admin_src, admin_start = _find_source(admin_conf, t_start) \ - if admin_conf else (None, None) - - if conf_id == admin_conf or admin_conf is None: - # Admin interval: show admin webcam full size - src_path, seg_start = _find_source(conf_id, t_start) - if src_path is None: - # Fallback: try any conf - for cid, segs in conf_segs.items(): - for path, ss, sd, se in segs: - if ss <= t_start + 0.5 and se >= t_start + 0.5: - src_path, seg_start = path, ss - break - if src_path: + if source is None: + # No segment for this conf_id at this time — try any conf + for cid, segs in conf_segs.items(): + for path, seg_start, seg_dur, seg_end in segs: + if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: + source = (path, seg_start) break - if src_path is None: - continue - offset = t_start - seg_start - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, offset)), - '-i', src_path, - '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, admin)', - ) - else: - # Non-admin interval: admin top + participant bottom, mixed audio - participant_src, participant_start = _find_source(conf_id, t_start) - if participant_src is None: - # No participant source — just use admin - if admin_src is None: - continue - offset = t_start - admin_start - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, offset)), - '-i', admin_src, - '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, admin fallback)', - ) - elif admin_src is not None: - # Admin top + participant bottom, mix ALL available audio - p_offset = t_start - participant_start - a_offset = t_start - admin_start - - # Find all webcam audio sources at this time - all_sources = _find_all_sources(t_start) - - # Build inputs: 0=admin(video+audio), 1=participant(video+audio), - # 2..N=extra audio sources - inputs = [ - '-ss', str(max(0, a_offset)), '-i', admin_src, - '-ss', str(max(0, p_offset)), '-i', participant_src, - ] - extra_audio = [] - for src_path_extra, src_offset in all_sources: - if src_path_extra in (admin_src, participant_src): - continue - idx = len(inputs) // 4 + len(extra_audio) + 2 - inputs.extend(['-ss', str(max(0, src_offset)), - '-i', src_path_extra]) - extra_audio.append(idx) - - # Build audio mix filter - n_audio = 2 + len(extra_audio) - audio_labels = '[0:a][1:a]' + ''.join( - f'[{idx}:a]' for idx in extra_audio - ) - audio_filter = ( - f'{audio_labels}amix=inputs={n_audio}' - f':duration=first:normalize=0[a]' - ) + if source: + break - vfilter = ( - f'[0:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[admin];' - f'[1:v]scale={target_w}:{half_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{half_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[part];' - f'[admin][part]vstack[v];' - f'{audio_filter}' - ) + if source is None: + continue - try: - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - *inputs, - '-t', str(duration), - '-filter_complex', vfilter, - '-map', '[v]', '-map', '[a]', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, admin+participant+{len(extra_audio)} extra)', - ) - except subprocess.CalledProcessError: - # Fallback: admin only if complex mix fails - logging.warning(f'Multi-audio failed for segment {i}, using admin only') - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, a_offset)), - '-i', admin_src, - '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, admin fallback)', - ) - else: - # No admin source — participant + all other audio - p_offset = t_start - participant_start - all_sources = _find_all_sources(t_start) - inputs = ['-ss', str(max(0, p_offset)), '-i', participant_src] - extra_audio = [] - for src_path_extra, src_offset in all_sources: - if src_path_extra == participant_src: - continue - idx = 1 + len(extra_audio) - inputs.extend(['-ss', str(max(0, src_offset)), - '-i', src_path_extra]) - extra_audio.append(idx) - - if extra_audio: - n_audio = 1 + len(extra_audio) - audio_labels = '[0:a]' + ''.join( - f'[{idx}:a]' for idx in extra_audio - ) - audio_filter = ( - f'{audio_labels}amix=inputs={n_audio}' - f':duration=first:normalize=0[a]' - ) - vf = ( - f'[0:v]scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[v];' - f'{audio_filter}' - ) - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - *inputs, '-t', str(duration), - '-filter_complex', vf, - '-map', '[v]', '-map', '[a]', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, participant+{len(extra_audio)} extra)', - ) - else: - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - *inputs, '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, participant only)', - ) + src_path, seg_start = source + offset = t_start - seg_start + seg_path = os.path.join(speaker_dir, f'speaker_{i:04d}.mp4') + + _run_ffmpeg( + [ + 'ffmpeg', '-y', '-v', 'error', + '-ss', str(max(0, offset)), + '-i', src_path, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + seg_path, + ], + description=f'speaker segment {i+1}/{len(speaker_timeline)} ' + f'({duration:.0f}s, conf {conf_id})', + ) result.append((seg_path, t_start)) logging.info(f'Speaker switching: built {len(result)} segments') @@ -1394,15 +1211,15 @@ def compile_final_video( break if has_overlaps and slide_events: - # Active speaker switching: show the talking person's webcam - logging.info('Multiple concurrent webcams + slides, using speaker switching') - speaker_timeline, admin_conf = _build_speaker_timeline( - video_files, total_duration, - ) - video_files = _build_speaker_switched_segments( - video_files, speaker_timeline, admin_conf, - tmp_dir, target_w, target_h, - ) + # Keep admin webcam, move other webcam audio to audio_files + logging.info('Multiple concurrent webcams + slides, keeping admin webcam') + deduped = _deduplicate_overlapping(video_files) + # Find webcam files that were dropped — add their audio to the mix + kept_paths = {v[0] for v in deduped} + for vpath, start_time, *_ in video_files: + if vpath not in kept_paths: + audio_files.append((vpath, start_time)) + video_files = deduped elif has_overlaps: # Grid layout: composite all concurrent webcams into a grid logging.info('Multiple concurrent webcams detected, building grid layout') From 9eb487dee80ad1d97a43ee9af0b084d09e01ef48 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 8 Apr 2026 06:07:20 +0000 Subject: [PATCH 38/65] Only add webcam files with audio streams to audio mix Webcam video files without audio caused 'matches no streams' errors in the amix filter. Now checks for audio stream before adding. --- mtslinker/processor.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index b45f87e..59ebd2d 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1216,9 +1216,18 @@ def compile_final_video( deduped = _deduplicate_overlapping(video_files) # Find webcam files that were dropped — add their audio to the mix kept_paths = {v[0] for v in deduped} + extra_audio = 0 for vpath, start_time, *_ in video_files: if vpath not in kept_paths: - audio_files.append((vpath, start_time)) + # Only add if the file has an audio stream + info = _ffprobe_streams(vpath) + has_audio = any( + s.get('codec_type') == 'audio' for s in info.get('streams', []) + ) + if has_audio: + audio_files.append((vpath, start_time)) + extra_audio += 1 + logging.info(f'Added {extra_audio} webcam audio tracks to mix') video_files = deduped elif has_overlaps: # Grid layout: composite all concurrent webcams into a grid From 9ac85a29ff440b548c3e10c1db4d39ca8eb30eee Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 8 Apr 2026 08:28:06 +0000 Subject: [PATCH 39/65] Extract audio from webcam files before adding to mix Video+audio files in the amix pipeline caused 'no streams' and 'invalid argument' errors. Now extracts audio to .m4a first with -vn, ensuring clean audio-only inputs for the merge step. --- mtslinker/processor.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 59ebd2d..13f93e6 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1214,19 +1214,33 @@ def compile_final_video( # Keep admin webcam, move other webcam audio to audio_files logging.info('Multiple concurrent webcams + slides, keeping admin webcam') deduped = _deduplicate_overlapping(video_files) - # Find webcam files that were dropped — add their audio to the mix + # Find webcam files that were dropped — extract their audio for mixing kept_paths = {v[0] for v in deduped} + extra_dir = os.path.join(tmp_dir, 'extra_audio') + os.makedirs(extra_dir, exist_ok=True) extra_audio = 0 for vpath, start_time, *_ in video_files: if vpath not in kept_paths: - # Only add if the file has an audio stream info = _ffprobe_streams(vpath) has_audio = any( s.get('codec_type') == 'audio' for s in info.get('streams', []) ) if has_audio: - audio_files.append((vpath, start_time)) - extra_audio += 1 + # Extract audio only (strip video) so amix works cleanly + audio_path = os.path.join(extra_dir, f'extra_{extra_audio}.m4a') + try: + _run_ffmpeg( + ['ffmpeg', '-y', '-v', 'error', + '-i', vpath, '-vn', + '-c:a', 'aac', '-b:a', '128k', + '-ar', '44100', '-ac', '2', + audio_path], + description=f'extract audio from webcam {extra_audio}', + ) + audio_files.append((audio_path, start_time)) + extra_audio += 1 + except subprocess.CalledProcessError: + pass # skip files that fail extraction logging.info(f'Added {extra_audio} webcam audio tracks to mix') video_files = deduped elif has_overlaps: From f3d1f62f90611fa67c517a2b36e56865ffad7cd1 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 8 Apr 2026 10:04:28 +0000 Subject: [PATCH 40/65] Refactor into two-phase pipeline: analyze then execute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 (analyze_video): Probes all files in parallel, makes all decisions (dedup, gap placement, audio extraction plan), writes a manifest.json. Fast — only ffprobe, no re-encoding. Phase 2 (execute_video): Reads manifest, executes ffmpeg commands. No probing, no decisions. Errors in phase 1 are caught early before spending hours on processing. compile_final_video is now a thin wrapper calling both phases. --- mtslinker/processor.py | 500 +++++++++++++++++++++++++++++------------ 1 file changed, 355 insertions(+), 145 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 13f93e6..ae01560 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1130,237 +1130,447 @@ def _composite_slides( return output_path -def compile_final_video( +def _probe_file(file_path: str) -> dict: + """Probe a single file and return all metadata needed for planning.""" + info = _ffprobe_streams(file_path) + streams = info.get('streams', []) + fmt = info.get('format', {}) + + has_video = any(s.get('codec_type') == 'video' for s in streams) + has_audio = any(s.get('codec_type') == 'audio' for s in streams) + + width, height, pix_fmt = 0, 0, 'yuv420p' + for s in streams: + if s.get('codec_type') == 'video': + width = int(s.get('width', 0)) + height = int(s.get('height', 0)) + pix_fmt = s.get('pix_fmt', 'yuv420p') + break + + duration = 0.0 + if 'duration' in fmt: + duration = float(fmt['duration']) + else: + for s in streams: + if 'duration' in s: + duration = float(s['duration']) + break + + valid = bool(streams) and duration > 0 + + return { + 'path': file_path, + 'valid': valid, + 'has_video': has_video, + 'has_audio': has_audio, + 'width': width, + 'height': height, + 'pix_fmt': pix_fmt, + 'duration': duration, + } + + +def _probe_all_files(downloaded_files: list) -> List[dict]: + """Probe all files in parallel. Returns list of file info dicts.""" + results = [] + paths = [(item[0], item[1], + item[2] if len(item) > 2 else None, + item[3] if len(item) > 3 else False) + for item in downloaded_files] + + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {} + for path, start_time, conf_id, is_admin in paths: + fut = pool.submit(_probe_file, path) + futures[fut] = (start_time, conf_id, is_admin) + for fut in as_completed(futures): + start_time, conf_id, is_admin = futures[fut] + info = fut.result() + info['start_time'] = start_time + info['conf_id'] = conf_id + info['is_admin'] = is_admin + results.append(info) + + results.sort(key=lambda x: x['start_time']) + return results + + +def analyze_video( total_duration: float, - downloaded_files: List[Tuple[str, float]], + downloaded_files: list, directory: str, output_path: str, - max_duration: Union[int, None], - slide_events: List[Dict] = None, -): - """Concatenate downloaded segments using ffmpeg (no MoviePy re-encoding). - - Steps: - 1. Separate video and audio-only files. - 2. Normalize video segments to a common resolution. - 3. Generate black gap segments where needed. - 4. Concatenate with ffmpeg concat demuxer (-c copy). - 5. Merge audio-only tracks on top (in batches to avoid OOM). + max_duration=None, + slide_events=None, +) -> dict: + """Phase 1: Probe all files, make all decisions, write manifest. + + Fast — only ffprobe calls, no re-encoding. Returns manifest dict + and writes it to {directory}/manifest.json. """ _check_ffmpeg() + logging.info('Phase 1: Analyzing files...') - video_files = [] # (path, start_time, conf_id, is_admin) - audio_files = [] # (path, start_time) + # Probe all files in parallel + all_files = _probe_all_files(downloaded_files) + + # Classify + video_files = [] # dicts with has_video=True + audio_files = [] # dicts with has_video=False (audio-only) skipped = 0 + errors = [] - for item in downloaded_files: - file_path, start_time = item[0], item[1] - conf_id = item[2] if len(item) > 2 else None - is_admin = item[3] if len(item) > 3 else False - if not _is_valid_media(file_path): + for f in all_files: + if not f['valid']: skipped += 1 + errors.append(f'Corrupt/invalid: {f["path"]}') continue - if _has_video_stream(file_path): - video_files.append((file_path, start_time, conf_id, is_admin)) + if f['has_video']: + video_files.append(f) else: - audio_files.append((file_path, start_time)) + audio_files.append(f) if skipped: logging.warning(f'Skipped {skipped} corrupt/invalid files') logging.info(f'Segments: {len(video_files)} video, {len(audio_files)} audio-only') if not video_files: - logging.error('No video segments found.') - return + raise ValueError('No valid video segments found.') - # Determine target resolution from the first video segment - # Pick the largest resolution among video segments (some may be tiny thumbnails) + # Determine target resolution target_w, target_h, target_pix_fmt = 0, 0, 'yuv420p' - for vpath, *_ in video_files: - w, h, pf = _get_video_params(vpath) - if w * h > target_w * target_h: - target_w, target_h, target_pix_fmt = w, h, pf - # Minimum 640x360 for reasonable quality + for f in video_files: + if f['width'] * f['height'] > target_w * target_h: + target_w, target_h, target_pix_fmt = f['width'], f['height'], f['pix_fmt'] if target_w * target_h < 640 * 360: target_w, target_h = 640, 360 - # Cap at 720p to avoid excessive memory use MAX_H = 720 if target_h > MAX_H: target_w = int(target_w * MAX_H / target_h) - target_w -= target_w % 2 # keep even + target_w -= target_w % 2 target_h = MAX_H logging.info(f'Target resolution: {target_w}x{target_h}, pix_fmt={target_pix_fmt}') - # Sort by start time - video_files.sort(key=lambda x: x[1]) - - # Build the list of segments (normalized videos + gap fillers) - tmp_dir = os.path.join(directory, '_tmp_ffmpeg') - os.makedirs(tmp_dir, exist_ok=True) - - # Check for overlapping segments (multiple concurrent webcams) + # Detect overlaps using cached durations has_overlaps = False - annotated_check = sorted( - [(item[0], item[1], _get_duration(item[0])) for item in video_files], - key=lambda x: x[1], - ) - for i in range(1, len(annotated_check)): - prev_end = annotated_check[i-1][1] + annotated_check[i-1][2] - if annotated_check[i][1] < prev_end - 0.5: + for i in range(1, len(video_files)): + prev_end = video_files[i-1]['start_time'] + video_files[i-1]['duration'] + if video_files[i]['start_time'] < prev_end - 0.5: has_overlaps = True break + # Decide overlap strategy and build segment plan + overlap_strategy = 'none' + kept_video = [] # files to use as video segments + extra_audio = [] # webcam files dropped but with audio to extract + if has_overlaps and slide_events: - # Keep admin webcam, move other webcam audio to audio_files - logging.info('Multiple concurrent webcams + slides, keeping admin webcam') - deduped = _deduplicate_overlapping(video_files) - # Find webcam files that were dropped — extract their audio for mixing + overlap_strategy = 'dedup' + # Use _deduplicate_overlapping with our probed data + vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), + f.get('is_admin', False)) for f in video_files] + deduped = _deduplicate_overlapping(vf_tuples) kept_paths = {v[0] for v in deduped} - extra_dir = os.path.join(tmp_dir, 'extra_audio') - os.makedirs(extra_dir, exist_ok=True) - extra_audio = 0 - for vpath, start_time, *_ in video_files: - if vpath not in kept_paths: - info = _ffprobe_streams(vpath) - has_audio = any( - s.get('codec_type') == 'audio' for s in info.get('streams', []) - ) - if has_audio: - # Extract audio only (strip video) so amix works cleanly - audio_path = os.path.join(extra_dir, f'extra_{extra_audio}.m4a') - try: - _run_ffmpeg( - ['ffmpeg', '-y', '-v', 'error', - '-i', vpath, '-vn', - '-c:a', 'aac', '-b:a', '128k', - '-ar', '44100', '-ac', '2', - audio_path], - description=f'extract audio from webcam {extra_audio}', - ) - audio_files.append((audio_path, start_time)) - extra_audio += 1 - except subprocess.CalledProcessError: - pass # skip files that fail extraction - logging.info(f'Added {extra_audio} webcam audio tracks to mix') - video_files = deduped + + # Map back to file info dicts + dedup_map = {f['path']: f for f in video_files} + for path, start_time in deduped: + if path in dedup_map: + kept_video.append(dedup_map[path]) + + # Find dropped webcams with audio + for f in video_files: + if f['path'] not in kept_paths and f['has_audio']: + extra_audio.append(f) + + logging.info(f'Dedup: kept {len(kept_video)}, ' + f'{len(extra_audio)} extra audio from webcams') elif has_overlaps: - # Grid layout: composite all concurrent webcams into a grid - logging.info('Multiple concurrent webcams detected, building grid layout') - video_files = _build_grid_segments( - video_files, tmp_dir, target_w, target_h, total_duration, - ) + overlap_strategy = 'grid' + # Grid handled in execute phase (needs ffmpeg) + kept_video = video_files # pass all through else: - video_files = _deduplicate_overlapping(video_files) - logging.info(f'After dedup/grid/speaker: {len(video_files)} segments') - - concat_segments = [] + vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), + f.get('is_admin', False)) for f in video_files] + deduped = _deduplicate_overlapping(vf_tuples) + dedup_map = {f['path']: f for f in video_files} + for path, start_time in deduped: + if path in dedup_map: + kept_video.append(dedup_map[path]) + + # Build segment plan (gaps + video segments in order) + segments = [] current_time = 0.0 + for i, f in enumerate(kept_video): + start_time = f['start_time'] - for i, (vpath, start_time) in enumerate(video_files): - # Skip segments whose start_time is before current_time (overlap) if start_time < current_time - 0.5: - logging.warning( - f'Segment {i} starts at {start_time:.1f}s but current_time ' - f'is {current_time:.1f}s — skipping overlapping segment' - ) + errors.append(f'Segment {i} overlaps at {start_time:.1f}s (current={current_time:.1f}s)') continue - # Insert black gap if needed gap = start_time - current_time - if gap > 0.1: # skip tiny gaps < 100ms - gap_path = os.path.join(tmp_dir, f'gap_{i}.mp4') - _generate_black_segment(gap_path, gap, target_w, target_h, target_pix_fmt) - concat_segments.append(gap_path) - logging.info(f'Generated {gap:.1f}s black gap before segment {i}') - - # Compute max allowed duration: truncate if this segment would - # overlap the next segment's start_time - max_dur = 0 # 0 = no limit - next_start = None - for j in range(i + 1, len(video_files)): - ns = video_files[j][1] + if gap > 0.1: + segments.append({ + 'type': 'gap', + 'duration': gap, + 'start_time': current_time, + }) + + # Compute max duration (truncate at next segment) + max_dur = 0 + for j in range(i + 1, len(kept_video)): + ns = kept_video[j]['start_time'] if ns > start_time + 0.5: - next_start = ns + max_dur = ns - start_time break - if next_start is not None: - max_dur = next_start - start_time - - # Normalize the segment (with optional truncation) - norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') - _normalize_segment(vpath, norm_path, target_w, target_h, target_pix_fmt, - max_duration=max_dur) - # Ensure it has an audio stream - with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') - final_seg = _ensure_audio_stream(norm_path, with_audio_path) - concat_segments.append(final_seg) - - # Use intended duration (from timeline) to advance current_time, - # not actual file duration which may drift due to re-encoding. + + segments.append({ + 'type': 'video', + 'source_path': f['path'], + 'start_time': start_time, + 'duration': f['duration'], + 'max_duration': max_dur, + 'has_audio': f['has_audio'], + }) + if max_dur > 0: current_time = start_time + max_dur - elif next_start is not None: - current_time = next_start else: - # Last segment — use actual duration as fallback - seg_dur = _get_duration(final_seg) - current_time = start_time + seg_dur + current_time = start_time + f['duration'] # Trailing gap if current_time < total_duration - 0.1: - gap_path = os.path.join(tmp_dir, 'gap_end.mp4') - _generate_black_segment(gap_path, total_duration - current_time, - target_w, target_h, target_pix_fmt) - concat_segments.append(gap_path) + segments.append({ + 'type': 'gap', + 'duration': total_duration - current_time, + 'start_time': current_time, + }) + + # Audio merge plan + all_audio = [{'path': f['path'], 'start_time': f['start_time'], + 'origin': 'original'} for f in audio_files] + for f in extra_audio: + all_audio.append({ + 'path': f['path'], + 'start_time': f['start_time'], + 'origin': 'webcam_extract', + }) + + # Slide compositing plan + slide_plan = None + if slide_events: + CHUNK_SECS = 1800 + n_chunks = max(1, int(total_duration + CHUNK_SECS - 1) // CHUNK_SECS) + slide_plan = { + 'events': slide_events, + 'chunk_seconds': CHUNK_SECS, + 'num_chunks': n_chunks, + } + + # GPU detection + _detect_gpu() + gpu = { + 'nvenc': bool(_NVENC_AVAILABLE), + 'cuda_overlay': bool(_CUDA_OVERLAY_AVAILABLE), + } + + manifest = { + 'version': 1, + 'total_duration': total_duration, + 'directory': directory, + 'output_path': output_path, + 'max_duration': max_duration, + 'target': { + 'width': target_w, + 'height': target_h, + 'pix_fmt': target_pix_fmt, + }, + 'overlap_strategy': overlap_strategy, + 'segments': segments, + 'audio_tracks': all_audio, + 'slide_compositing': slide_plan, + 'gpu': gpu, + 'errors': errors, + 'warnings': [], + 'stats': { + 'total_files': len(all_files), + 'video_files': len(video_files), + 'audio_files': len(audio_files), + 'kept_video': len(kept_video), + 'extra_audio': len(extra_audio), + 'skipped': skipped, + 'segments': len(segments), + 'gaps': sum(1 for s in segments if s['type'] == 'gap'), + }, + } + + # Write manifest + manifest_path = os.path.join(directory, 'manifest.json') + with open(manifest_path, 'w') as f: + json.dump(manifest, f, indent=2, ensure_ascii=False) + logging.info(f'Manifest written to {manifest_path}') + logging.info(f'Plan: {manifest["stats"]["segments"]} segments ' + f'({manifest["stats"]["gaps"]} gaps), ' + f'{len(all_audio)} audio tracks, ' + f'strategy={overlap_strategy}') + + if errors: + for e in errors: + logging.warning(f'Analyze: {e}') + + return manifest + + +def execute_video(manifest: dict): + """Phase 2: Execute the plan from the manifest. No probing, no decisions.""" + logging.info('Phase 2: Executing plan...') + + directory = manifest['directory'] + output_path = manifest['output_path'] + total_duration = manifest['total_duration'] + target = manifest['target'] + target_w, target_h = target['width'], target['height'] + target_pix_fmt = target['pix_fmt'] + + tmp_dir = os.path.join(directory, '_tmp_ffmpeg') + os.makedirs(tmp_dir, exist_ok=True) + + # Step 1: Extract audio from dropped webcam files + audio_files = [] # (path, start_time) for merge step + extra_dir = os.path.join(tmp_dir, 'extra_audio') + os.makedirs(extra_dir, exist_ok=True) + extract_count = 0 + for track in manifest['audio_tracks']: + if track['origin'] == 'webcam_extract': + audio_path = os.path.join(extra_dir, f'extra_{extract_count}.m4a') + try: + _run_ffmpeg( + ['ffmpeg', '-y', '-v', 'error', + '-i', track['path'], '-vn', + '-c:a', 'aac', '-b:a', '128k', + '-ar', '44100', '-ac', '2', + audio_path], + description=f'extract audio from webcam {extract_count}', + ) + audio_files.append((audio_path, track['start_time'])) + extract_count += 1 + except subprocess.CalledProcessError: + logging.warning(f'Failed to extract audio from {track["path"]}') + else: + audio_files.append((track['path'], track['start_time'])) + if extract_count: + logging.info(f'Extracted audio from {extract_count} webcam files') + + # Step 2: Handle grid layout if needed (requires ffmpeg) + segments = manifest['segments'] + if manifest['overlap_strategy'] == 'grid': + # Grid needs full video_files — rebuild from manifest + # For now, pass through to the old grid function + logging.info('Grid layout — using legacy grid builder') + vf_tuples = [] + for seg in segments: + if seg['type'] == 'video': + vf_tuples.append((seg['source_path'], seg['start_time'])) + video_files = _build_grid_segments( + [(p, s, None, False) for p, s in vf_tuples], + tmp_dir, target_w, target_h, total_duration, + ) + # Rebuild segments from grid output + segments = [] + current_time = 0.0 + for vpath, start_time in video_files: + gap = start_time - current_time + if gap > 0.1: + segments.append({'type': 'gap', 'duration': gap, + 'start_time': current_time}) + segments.append({ + 'type': 'video', 'source_path': vpath, + 'start_time': start_time, + 'duration': _get_duration(vpath), + 'max_duration': 0, 'has_audio': True, + }) + current_time = start_time + _get_duration(vpath) - # Write concat list + # Step 3: Normalize segments and generate gaps + concat_segments = [] + for i, seg in enumerate(segments): + if seg['type'] == 'gap': + gap_path = os.path.join(tmp_dir, f'gap_{i}.mp4') + _generate_black_segment(gap_path, seg['duration'], + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + logging.info(f'Generated {seg["duration"]:.1f}s black gap') + elif seg['type'] == 'video': + norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') + _normalize_segment(seg['source_path'], norm_path, + target_w, target_h, target_pix_fmt, + max_duration=seg.get('max_duration', 0)) + with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') + final_seg = _ensure_audio_stream(norm_path, with_audio_path) + concat_segments.append(final_seg) + + # Step 4: Concatenate concat_list_path = os.path.join(tmp_dir, 'concat.txt') with open(concat_list_path, 'w') as f: for seg in concat_segments: f.write(f"file '{os.path.abspath(seg)}'\n") - # Concatenate with ffmpeg concat demuxer (stream copy - no re-encoding) video_only_path = os.path.join(tmp_dir, 'video_concat.mp4') logging.info(f'Concatenating {len(concat_segments)} segments...') + cap = manifest.get('max_duration') or total_duration concat_cmd = [ 'ffmpeg', '-y', '-v', 'warning', '-f', 'concat', '-safe', '0', '-i', concat_list_path, '-c', 'copy', ] - - # Cap at max_duration or total_duration to prevent inflated output - cap = max_duration or total_duration if cap: concat_cmd.extend(['-t', str(cap)]) - concat_cmd.append(video_only_path) _run_ffmpeg(concat_cmd, description=f'concat {len(concat_segments)} segments') - # Composite slides if presentation data exists - if slide_events: + # Step 5: Composite slides + slide_plan = manifest.get('slide_compositing') + if slide_plan and slide_plan.get('events'): composited_path = os.path.join(tmp_dir, 'video_composited.mp4') _composite_slides( - video_only_path, slide_events, composited_path, + video_only_path, slide_plan['events'], composited_path, tmp_dir, total_duration, ) video_only_path = composited_path - # If there are audio-only tracks, overlay them + # Step 6: Merge audio if audio_files: - logging.info(f'Merging {len(audio_files)} audio-only tracks...') + logging.info(f'Merging {len(audio_files)} audio tracks...') result_path = _merge_audio_tracks( video_only_path, audio_files, tmp_dir, output_path, total_duration, ) else: - # Just move/copy the result shutil.move(video_only_path, output_path) result_path = output_path - # Cleanup temp files + # Cleanup shutil.rmtree(tmp_dir, ignore_errors=True) logging.info(f'Final video saved to {result_path}') + return result_path + + +def compile_final_video( + total_duration: float, + downloaded_files: List[Tuple[str, float]], + directory: str, + output_path: str, + max_duration: Union[int, None], + slide_events: List[Dict] = None, +): + """Two-phase video compilation: analyze then execute.""" + manifest = analyze_video( + total_duration, downloaded_files, directory, output_path, + max_duration, slide_events, + ) + execute_video(manifest) def _merge_audio_tracks( From 2053d7c0d85badb6254e3c0b3bf3192f57d0e552 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 8 Apr 2026 13:37:44 +0000 Subject: [PATCH 41/65] Fix video/audio timeline mismatch: pad short segments, validate manifest Phase 1: Added planned_duration and source dimensions to manifest. Validates segments where source is shorter than planned and warns. Phase 2: After normalizing each segment, checks actual duration vs planned. Pads with black if too short. This ensures the concat video matches total_duration exactly, so adelay-based audio placement stays in sync. --- mtslinker/processor.py | 47 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index ae01560..0843c8f 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1324,19 +1324,21 @@ def analyze_video( max_dur = ns - start_time break + # planned_duration = how long this segment should be in the output + planned_dur = max_dur if max_dur > 0 else f['duration'] segments.append({ 'type': 'video', 'source_path': f['path'], 'start_time': start_time, - 'duration': f['duration'], + 'source_duration': f['duration'], 'max_duration': max_dur, + 'planned_duration': planned_dur, 'has_audio': f['has_audio'], + 'width': f['width'], + 'height': f['height'], }) - if max_dur > 0: - current_time = start_time + max_dur - else: - current_time = start_time + f['duration'] + current_time = start_time + planned_dur # Trailing gap if current_time < total_duration - 0.1: @@ -1414,9 +1416,26 @@ def analyze_video( f'{len(all_audio)} audio tracks, ' f'strategy={overlap_strategy}') + # Validate: check for segments where source is shorter than planned + warnings = [] + for seg in segments: + if seg['type'] != 'video': + continue + src_dur = seg['source_duration'] + planned = seg['planned_duration'] + if src_dur < planned - 1.0: + warnings.append( + f'Segment at {seg["start_time"]:.0f}s: source={src_dur:.0f}s ' + f'but planned={planned:.0f}s (will pad {planned-src_dur:.0f}s black)' + ) + manifest['warnings'] = warnings + if errors: for e in errors: logging.warning(f'Analyze: {e}') + if warnings: + for w in warnings: + logging.warning(f'Analyze: {w}') return manifest @@ -1507,7 +1526,23 @@ def execute_video(manifest: dict): max_duration=seg.get('max_duration', 0)) with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') final_seg = _ensure_audio_stream(norm_path, with_audio_path) - concat_segments.append(final_seg) + + # Verify duration matches plan — pad with black if too short + actual_dur = _get_duration(final_seg) + planned_dur = seg.get('planned_duration', 0) + if planned_dur > 0 and actual_dur < planned_dur - 1.0: + shortfall = planned_dur - actual_dur + logging.warning( + f'Segment {i} is {actual_dur:.1f}s but planned ' + f'{planned_dur:.1f}s — padding {shortfall:.1f}s' + ) + pad_path = os.path.join(tmp_dir, f'pad_{i}.mp4') + _generate_black_segment(pad_path, shortfall, + target_w, target_h, target_pix_fmt) + concat_segments.append(final_seg) + concat_segments.append(pad_path) + else: + concat_segments.append(final_seg) # Step 4: Concatenate concat_list_path = os.path.join(tmp_dir, 'concat.txt') From 8479c2fb01d06fe3c76d20fa0c89dd9b481417bd Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 9 Apr 2026 07:46:37 +0000 Subject: [PATCH 42/65] Move grid planning to phase 1, handle failures gracefully in phase 2 Grid windows are now planned in analyze_video using probed durations. Sources with <0.5s remaining at seek position are excluded upfront. Execute phase handles grid segment failures by falling back to black. Grid segments are stored in manifest with type='grid' and validated source lists. --- mtslinker/processor.py | 139 ++++++++++++++++++++++++++++++----------- 1 file changed, 104 insertions(+), 35 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 0843c8f..9630675 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1263,6 +1263,7 @@ def analyze_video( overlap_strategy = 'none' kept_video = [] # files to use as video segments extra_audio = [] # webcam files dropped but with audio to extract + segments = [] # populated by grid, or built later for dedup/none if has_overlaps and slide_events: overlap_strategy = 'dedup' @@ -1287,8 +1288,72 @@ def analyze_video( f'{len(extra_audio)} extra audio from webcams') elif has_overlaps: overlap_strategy = 'grid' - # Grid handled in execute phase (needs ffmpeg) - kept_video = video_files # pass all through + # Plan grid windows using probed durations + # Build annotated list: (path, start, duration, end) + annotated = [(f['path'], f['start_time'], f['duration'], + f['start_time'] + f['duration']) for f in video_files] + # Collect all event times + event_times_set = set() + for _, start, _, end in annotated: + event_times_set.add(start) + event_times_set.add(end) + event_times_set.add(total_duration) + event_times = sorted(event_times_set) + + # Build grid windows with validated sources + grid_windows = [] + prev_active_key = None + window_start = None + window_active = None + for ei in range(len(event_times) - 1): + t_start = event_times[ei] + t_end = event_times[ei + 1] + if t_end - t_start < 0.1: + continue + active = [] + for path, seg_start, dur, seg_end in annotated: + if seg_start < t_end and seg_end > t_start: + offset = max(0, t_start - seg_start) + remaining = dur - offset + if remaining > 0.5: # only include if >0.5s remaining + active.append({'path': path, 'offset': offset, + 'remaining': remaining}) + active_key = tuple(a['path'] for a in active) + if active_key == prev_active_key and window_start is not None: + continue + if prev_active_key is not None and window_start is not None: + grid_windows.append({ + 'start_time': window_start, + 'duration': t_start - window_start, + 'sources': window_active, + }) + window_start = t_start + prev_active_key = active_key + window_active = active + if window_start is not None and window_active: + grid_windows.append({ + 'start_time': window_start, + 'duration': event_times[-1] - window_start, + 'sources': window_active, + }) + + # Convert grid windows to segments + for gw in grid_windows: + if not gw['sources']: + segments.append({ + 'type': 'gap', + 'duration': gw['duration'], + 'start_time': gw['start_time'], + }) + else: + segments.append({ + 'type': 'grid', + 'start_time': gw['start_time'], + 'planned_duration': gw['duration'], + 'sources': gw['sources'], + }) + # No extra audio extraction for grid (audio handled by merge) + # kept_video not used for grid — segments has everything else: vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), f.get('is_admin', False)) for f in video_files] @@ -1299,7 +1364,9 @@ def analyze_video( kept_video.append(dedup_map[path]) # Build segment plan (gaps + video segments in order) - segments = [] + # Grid strategy already has segments populated + if overlap_strategy != 'grid': + segments = [] current_time = 0.0 for i, f in enumerate(kept_video): start_time = f['start_time'] @@ -1340,8 +1407,8 @@ def analyze_video( current_time = start_time + planned_dur - # Trailing gap - if current_time < total_duration - 0.1: + # Trailing gap (not for grid — grid segments already cover full timeline) + if overlap_strategy != 'grid' and current_time < total_duration - 0.1: segments.append({ 'type': 'gap', 'duration': total_duration - current_time, @@ -1480,37 +1547,9 @@ def execute_video(manifest: dict): if extract_count: logging.info(f'Extracted audio from {extract_count} webcam files') - # Step 2: Handle grid layout if needed (requires ffmpeg) + # Step 2: Process segments segments = manifest['segments'] - if manifest['overlap_strategy'] == 'grid': - # Grid needs full video_files — rebuild from manifest - # For now, pass through to the old grid function - logging.info('Grid layout — using legacy grid builder') - vf_tuples = [] - for seg in segments: - if seg['type'] == 'video': - vf_tuples.append((seg['source_path'], seg['start_time'])) - video_files = _build_grid_segments( - [(p, s, None, False) for p, s in vf_tuples], - tmp_dir, target_w, target_h, total_duration, - ) - # Rebuild segments from grid output - segments = [] - current_time = 0.0 - for vpath, start_time in video_files: - gap = start_time - current_time - if gap > 0.1: - segments.append({'type': 'gap', 'duration': gap, - 'start_time': current_time}) - segments.append({ - 'type': 'video', 'source_path': vpath, - 'start_time': start_time, - 'duration': _get_duration(vpath), - 'max_duration': 0, 'has_audio': True, - }) - current_time = start_time + _get_duration(vpath) - - # Step 3: Normalize segments and generate gaps + grid_dir = os.path.join(tmp_dir, 'grid_segments') concat_segments = [] for i, seg in enumerate(segments): if seg['type'] == 'gap': @@ -1519,6 +1558,36 @@ def execute_video(manifest: dict): target_w, target_h, target_pix_fmt) concat_segments.append(gap_path) logging.info(f'Generated {seg["duration"]:.1f}s black gap') + elif seg['type'] == 'grid': + os.makedirs(grid_dir, exist_ok=True) + seg_path = os.path.join(grid_dir, f'grid_{i}.mp4') + duration = seg['planned_duration'] + active = [(s['path'], s['offset']) for s in seg['sources']] + try: + if len(active) == 1: + path, offset = active[0] + _run_ffmpeg( + ['ffmpeg', '-y', '-v', 'error', + '-ss', str(offset), '-i', path, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *_get_video_encoder_fast(), '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', seg_path], + description=f'grid {i} (1 webcam, {duration:.0f}s)', + ) + else: + _composite_grid(active, duration, seg_path, target_w, target_h) + with_audio = os.path.join(tmp_dir, f'grida_{i}.mp4') + final_seg = _ensure_audio_stream(seg_path, with_audio) + concat_segments.append(final_seg) + except subprocess.CalledProcessError: + logging.warning(f'Grid segment {i} failed, using black gap') + gap_path = os.path.join(tmp_dir, f'gridfail_{i}.mp4') + _generate_black_segment(gap_path, duration, + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) elif seg['type'] == 'video': norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') _normalize_segment(seg['source_path'], norm_path, From a00246ffc2ed862d342e34dcf38bf0e2c6e7211a Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 9 Apr 2026 14:00:14 +0000 Subject: [PATCH 43/65] Include audio in grid composite, add has_audio to grid sources Grid composite was using -an (no audio), losing the presenter's voice for the entire recording. Now uses -map 0:a? to include audio from the first input. Grid sources are sorted so audio-bearing inputs come first. has_audio from probe data is stored in manifest. --- mtslinker/processor.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 9630675..96f2cca 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -587,16 +587,16 @@ def _composite_grid( + f'xstack=inputs={total_cells}:layout={layout}[out]' ) - # Audio is handled separately by _merge_audio_tracks — output video only. + # Include audio from first input if available (? = optional) cmd = [ 'ffmpeg', '-y', '-v', 'error', *inputs, '-t', str(duration), '-filter_complex', filter_graph, - '-map', '[out]', - '-an', + '-map', '[out]', '-map', '0:a?', *_get_video_encoder_fast(), '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', '-r', '25', output_path, ] @@ -1291,7 +1291,8 @@ def analyze_video( # Plan grid windows using probed durations # Build annotated list: (path, start, duration, end) annotated = [(f['path'], f['start_time'], f['duration'], - f['start_time'] + f['duration']) for f in video_files] + f['start_time'] + f['duration'], f['has_audio']) + for f in video_files] # Collect all event times event_times_set = set() for _, start, _, end in annotated: @@ -1311,13 +1312,14 @@ def analyze_video( if t_end - t_start < 0.1: continue active = [] - for path, seg_start, dur, seg_end in annotated: + for path, seg_start, dur, seg_end, has_audio in annotated: if seg_start < t_end and seg_end > t_start: offset = max(0, t_start - seg_start) remaining = dur - offset - if remaining > 0.5: # only include if >0.5s remaining + if remaining > 0.5: active.append({'path': path, 'offset': offset, - 'remaining': remaining}) + 'remaining': remaining, + 'has_audio': has_audio}) active_key = tuple(a['path'] for a in active) if active_key == prev_active_key and window_start is not None: continue @@ -1562,7 +1564,10 @@ def execute_video(manifest: dict): os.makedirs(grid_dir, exist_ok=True) seg_path = os.path.join(grid_dir, f'grid_{i}.mp4') duration = seg['planned_duration'] - active = [(s['path'], s['offset']) for s in seg['sources']] + # Sort sources: audio-bearing first so input 0 has audio + sources = sorted(seg['sources'], + key=lambda s: not s.get('has_audio', False)) + active = [(s['path'], s['offset']) for s in sources] try: if len(active) == 1: path, offset = active[0] From 54a27a496f8b3b35a562008847adf69b961e6c18 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 9 Apr 2026 14:19:39 +0000 Subject: [PATCH 44/65] Fix tuple unpack for 5-element annotated grid data --- mtslinker/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 96f2cca..c3f3061 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1295,7 +1295,7 @@ def analyze_video( for f in video_files] # Collect all event times event_times_set = set() - for _, start, _, end in annotated: + for _, start, _, end, _ in annotated: event_times_set.add(start) event_times_set.add(end) event_times_set.add(total_duration) From 25fa3b406228be793579e0cc13320dbc36c52bf1 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 9 Apr 2026 17:28:20 +0000 Subject: [PATCH 45/65] Use API mediasession duration for accurate timing, include has_audio in grid - Parse mediasession.add/update events for precise file durations - Use api_duration (when available) instead of ffprobe for segment planning and overlap detection - Include api_duration in tagged_chunks for downstream use - Grid sources track has_audio, sorted so audio-bearing input is first - Grid composite uses 0:a? to preserve presenter audio --- mtslinker/processor.py | 74 +++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index c3f3061..e55634b 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -786,27 +786,55 @@ def process_and_download_clips( admin_conf_ids = _extract_admin_conf_ids(json_data) + # Parse mediasession events for precise timing + mediasession_adds = {} # id -> {url, start, conf_id} chunks = [] slide_events = [] for event in json_data.get('eventLogs', []): if not isinstance(event, dict): continue + module = event.get('module', '') data = event.get('data', {}) if not isinstance(data, dict): continue - # Media chunks (video/audio) - if 'url' in data: + # mediasession.add: precise file start time + if module == 'mediasession.add' and 'url' in data: + ms_id = data.get('id') + stream = data.get('stream', {}) + conf_id = None + if isinstance(stream, dict): + conf = stream.get('conference', {}) + if isinstance(conf, dict): + conf_id = conf.get('id') + mediasession_adds[ms_id] = { + 'url': data['url'], + 'start': event.get('relativeTime', 0), + 'conf_id': conf_id, + 'api_duration': 0, + } + + # mediasession.update: duration + if module == 'mediasession.update': + ms_id = data.get('id') + if ms_id in mediasession_adds: + mediasession_adds[ms_id]['api_duration'] = data.get('duration', 0) + + # Fallback: any event with URL (catches non-mediasession chunks) + elif 'url' in data and module not in ('mediasession.add',): stream = data.get('stream', {}) conf_id = None if isinstance(stream, dict): conf = stream.get('conference', {}) if isinstance(conf, dict): conf_id = conf.get('id') - chunks.append((data['url'], event.get('relativeTime', 0), conf_id)) + # Only add if not already in mediasession_adds + url = data['url'] + if not any(ms['url'] == url for ms in mediasession_adds.values()): + chunks.append((url, event.get('relativeTime', 0), conf_id, 0)) # Presentation slide changes - if event.get('module') == 'presentation.update': + if module == 'presentation.update': fr = data.get('fileReference', {}) if not isinstance(fr, dict): continue @@ -820,10 +848,16 @@ def process_and_download_clips( 'slide_url': slide_url, }) - # Tag chunks from admin conferences for dedup priority + # Build tagged chunks from mediasession data (preferred) + fallback chunks tagged_chunks = [] - for url, start, conf_id in chunks: - tagged_chunks.append((url, start, conf_id, conf_id in admin_conf_ids)) + for ms in mediasession_adds.values(): + tagged_chunks.append(( + ms['url'], ms['start'], ms['conf_id'], + ms['conf_id'] in admin_conf_ids, + ms['api_duration'], + )) + for url, start, conf_id, api_dur in chunks: + tagged_chunks.append((url, start, conf_id, conf_id in admin_conf_ids, api_dur)) # Deduplicate consecutive identical slides deduped_slides = [] @@ -1175,20 +1209,24 @@ def _probe_all_files(downloaded_files: list) -> List[dict]: results = [] paths = [(item[0], item[1], item[2] if len(item) > 2 else None, - item[3] if len(item) > 3 else False) + item[3] if len(item) > 3 else False, + item[4] if len(item) > 4 else 0) for item in downloaded_files] with ThreadPoolExecutor(max_workers=8) as pool: futures = {} - for path, start_time, conf_id, is_admin in paths: + for path, start_time, conf_id, is_admin, api_duration in paths: fut = pool.submit(_probe_file, path) - futures[fut] = (start_time, conf_id, is_admin) + futures[fut] = (start_time, conf_id, is_admin, api_duration) for fut in as_completed(futures): - start_time, conf_id, is_admin = futures[fut] + start_time, conf_id, is_admin, api_duration = futures[fut] info = fut.result() info['start_time'] = start_time info['conf_id'] = conf_id info['is_admin'] = is_admin + # Use API duration if available (more accurate than ffprobe) + if api_duration > 0: + info['api_duration'] = api_duration results.append(info) results.sort(key=lambda x: x['start_time']) @@ -1254,7 +1292,8 @@ def analyze_video( # Detect overlaps using cached durations has_overlaps = False for i in range(1, len(video_files)): - prev_end = video_files[i-1]['start_time'] + video_files[i-1]['duration'] + prev_dur = video_files[i-1].get('api_duration', 0) or video_files[i-1]['duration'] + prev_end = video_files[i-1]['start_time'] + prev_dur if video_files[i]['start_time'] < prev_end - 0.5: has_overlaps = True break @@ -1290,9 +1329,11 @@ def analyze_video( overlap_strategy = 'grid' # Plan grid windows using probed durations # Build annotated list: (path, start, duration, end) - annotated = [(f['path'], f['start_time'], f['duration'], - f['start_time'] + f['duration'], f['has_audio']) - for f in video_files] + annotated = [] + for f in video_files: + dur = f.get('api_duration', 0) or f['duration'] + annotated.append((f['path'], f['start_time'], dur, + f['start_time'] + dur, f['has_audio'])) # Collect all event times event_times_set = set() for _, start, _, end, _ in annotated: @@ -1394,7 +1435,8 @@ def analyze_video( break # planned_duration = how long this segment should be in the output - planned_dur = max_dur if max_dur > 0 else f['duration'] + file_dur = f.get('api_duration', 0) or f['duration'] + planned_dur = max_dur if max_dur > 0 else file_dur segments.append({ 'type': 'video', 'source_path': f['path'], From 6c3b62361554d466171f6f456c7024f3dcc2c419 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 9 Apr 2026 17:34:39 +0000 Subject: [PATCH 46/65] Add trailing black gap for grid recordings to preserve audio Grid recordings could end video at 6891s while audio continued to 11180s. The trailing audio was cut off. Now adds a black gap after the last grid segment to extend video to total_duration, so _merge_audio_tracks can overlay all audio including late segments. --- mtslinker/processor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index e55634b..7d20d31 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1451,8 +1451,15 @@ def analyze_video( current_time = start_time + planned_dur - # Trailing gap (not for grid — grid segments already cover full timeline) - if overlap_strategy != 'grid' and current_time < total_duration - 0.1: + # Trailing gap — extend video to total_duration so audio can play over black + if overlap_strategy == 'grid': + # For grid, compute current_time from segments + current_time = 0 + for s in segments: + end = s.get('start_time', 0) + s.get('planned_duration', s.get('duration', 0)) + if end > current_time: + current_time = end + if current_time < total_duration - 0.1: segments.append({ 'type': 'gap', 'duration': total_duration - current_time, From b4910965dfcf1eaff86fe22835b0cf9c46500ea7 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Thu, 9 Apr 2026 21:11:09 +0000 Subject: [PATCH 47/65] Fill grid timeline gaps in phase 1 to match audio timeline Grid segments could start late (e.g. 1762s) but concat placed them at 0s, causing video/audio desync. Now inserts leading and internal gaps in the manifest so the video timeline matches total_duration exactly. Validated: planned duration equals API duration with zero timeline gaps. --- mtslinker/processor.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 7d20d31..041ae86 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1451,9 +1451,24 @@ def analyze_video( current_time = start_time + planned_dur + # For grid: fill timeline gaps so video matches audio timeline exactly + if overlap_strategy == 'grid' and segments: + filled = [] + current_time = 0.0 + for s in segments: + start = s.get('start_time', 0) + if start - current_time > 0.1: + filled.append({ + 'type': 'gap', + 'duration': start - current_time, + 'start_time': current_time, + }) + filled.append(s) + current_time = start + s.get('planned_duration', s.get('duration', 0)) + segments = filled + # Trailing gap — extend video to total_duration so audio can play over black if overlap_strategy == 'grid': - # For grid, compute current_time from segments current_time = 0 for s in segments: end = s.get('start_time', 0) + s.get('planned_duration', s.get('duration', 0)) From 58053d3a7bac324b05edb438c0bc093d2dabbe81 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 10 Apr 2026 06:24:30 +0000 Subject: [PATCH 48/65] Extract webcam audio for grid recordings too Grid composite only keeps audio from input 0, losing all other webcam voices. Now extracts audio from all video files with audio streams for mixing in _merge_audio_tracks, same as dedup path. --- mtslinker/processor.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 041ae86..5fb37ce 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1395,7 +1395,13 @@ def analyze_video( 'planned_duration': gw['duration'], 'sources': gw['sources'], }) - # No extra audio extraction for grid (audio handled by merge) + # Extract audio from ALL video files with audio for mixing + # Grid only keeps audio from input 0 — other webcam audio is lost + for f in video_files: + if f['has_audio']: + extra_audio.append(f) + if extra_audio: + logging.info(f'Grid: {len(extra_audio)} webcam audio tracks to extract') # kept_video not used for grid — segments has everything else: vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), From 730c6b76e87060bf5cb66e61022e9851fcb7db29 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 10 Apr 2026 07:52:56 +0000 Subject: [PATCH 49/65] Refactor processor into composed classes with tests Split monolithic processor.py (1915 lines) into 6 focused classes: - FFmpegRunner: ffmpeg execution, GPU detection, encoder selection - MediaProber: file probing, duration, streams, audio levels - GridCompositor: multi-webcam grid layout - SlideCompositor: presentation slide overlay - AudioMerger: batched audio mixing with tree-reduce - SegmentBuilder: normalize, gaps, dedup, admin detection VideoProcessor composes all classes via constructor injection. No static methods, no underscore prefixes on public methods. processor.py is now a thin orchestrator with backward-compatible module-level functions. 33 tests across 5 test files, all passing. Added requirements-dev.txt with pytest. --- mtslinker/audio.py | 151 ++ mtslinker/ffmpeg.py | 80 + mtslinker/grid.py | 77 + mtslinker/prober.py | 179 ++ mtslinker/processor.py | 2304 ++++------------- mtslinker/segments.py | 192 ++ mtslinker/slides.py | 153 ++ requirements-dev.txt | 2 + tests/__init__.py | 0 tests/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 147 bytes .../conftest.cpython-311-pytest-9.0.3.pyc | Bin 0 -> 2701 bytes .../test_ffmpeg.cpython-311-pytest-9.0.3.pyc | Bin 0 -> 7805 bytes .../test_grid.cpython-311-pytest-9.0.3.pyc | Bin 0 -> 9633 bytes .../test_prober.cpython-311-pytest-9.0.3.pyc | Bin 0 -> 17714 bytes ...est_processor.cpython-311-pytest-9.0.3.pyc | Bin 0 -> 12284 bytes ...test_segments.cpython-311-pytest-9.0.3.pyc | Bin 0 -> 10455 bytes tests/conftest.py | 54 + tests/test_ffmpeg.py | 37 + tests/test_grid.py | 42 + tests/test_prober.py | 81 + tests/test_processor.py | 39 + tests/test_segments.py | 48 + 22 files changed, 1639 insertions(+), 1800 deletions(-) create mode 100644 mtslinker/audio.py create mode 100644 mtslinker/ffmpeg.py create mode 100644 mtslinker/grid.py create mode 100644 mtslinker/prober.py create mode 100644 mtslinker/segments.py create mode 100644 mtslinker/slides.py create mode 100644 requirements-dev.txt create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.cpython-311.pyc create mode 100644 tests/__pycache__/conftest.cpython-311-pytest-9.0.3.pyc create mode 100644 tests/__pycache__/test_ffmpeg.cpython-311-pytest-9.0.3.pyc create mode 100644 tests/__pycache__/test_grid.cpython-311-pytest-9.0.3.pyc create mode 100644 tests/__pycache__/test_prober.cpython-311-pytest-9.0.3.pyc create mode 100644 tests/__pycache__/test_processor.cpython-311-pytest-9.0.3.pyc create mode 100644 tests/__pycache__/test_segments.cpython-311-pytest-9.0.3.pyc create mode 100644 tests/conftest.py create mode 100644 tests/test_ffmpeg.py create mode 100644 tests/test_grid.py create mode 100644 tests/test_prober.py create mode 100644 tests/test_processor.py create mode 100644 tests/test_segments.py diff --git a/mtslinker/audio.py b/mtslinker/audio.py new file mode 100644 index 0000000..a1e0c03 --- /dev/null +++ b/mtslinker/audio.py @@ -0,0 +1,151 @@ +import logging +import os +import shutil +import subprocess +from typing import List, Tuple + +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber + + +class AudioMerger: + """Merges multiple audio tracks onto a video using batched amix.""" + + BATCH_SIZE = 8 + + def __init__(self, ffmpeg: FFmpegRunner, prober: MediaProber): + self.ffmpeg = ffmpeg + self.prober = prober + + def merge(self, video_path: str, audio_files: List[Tuple[str, float]], + tmp_dir: str, output_path: str, total_duration: float = 0) -> str: + video_duration = total_duration or self.prober.get_duration(video_path) + + # Step 1: Mix in batches with inline adelay + batch_outputs = [] + total_batches = (len(audio_files) + self.BATCH_SIZE - 1) // self.BATCH_SIZE + for batch_idx, batch_start in enumerate( + range(0, len(audio_files), self.BATCH_SIZE) + ): + batch = audio_files[batch_start:batch_start + self.BATCH_SIZE] + batch_out = os.path.join(tmp_dir, f'audio_batch_{batch_idx}.m4a') + + inputs = [] + filter_parts = [] + mix_labels = [] + for j, (apath, start_time) in enumerate(batch): + inputs.extend(['-i', apath]) + delay_ms = int(start_time * 1000) + label = f'a{j}' + filter_parts.append( + f'[{j}:a]adelay={delay_ms}|{delay_ms},' + f'apad=whole_dur={video_duration},' + f'atrim=0:{video_duration},' + f'asetpts=PTS-STARTPTS[{label}]' + ) + mix_labels.append(f'[{label}]') + + if len(batch) == 1: + filter_graph = filter_parts[0].rsplit('[', 1)[0] + else: + filter_graph = ( + ';'.join(filter_parts) + ';' + + ''.join(mix_labels) + + f'amix=inputs={len(batch)}:duration=longest:normalize=0' + ) + + try: + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-filter_complex', filter_graph, + '-c:a', 'aac', '-b:a', '128k', + '-ar', '44100', '-ac', '2', + batch_out, + ], + description=f'amix batch {batch_idx+1} ' + f'({len(batch)} tracks, offset {batch[0][1]:.0f}-' + f'{batch[-1][1]:.0f}s)', + ) + batch_outputs.append(batch_out) + except subprocess.CalledProcessError: + logging.warning( + f'Audio batch {batch_idx+1} failed, skipping {len(batch)} tracks' + ) + logging.info(f'Audio batch {batch_idx+1}/{total_batches} done') + + if not batch_outputs: + logging.warning('All audio batches failed, skipping audio overlay') + shutil.move(video_path, output_path) + return output_path + + # Step 2: Tree-reduce batch outputs + round_num = 0 + current_paths = batch_outputs + while len(current_paths) > 1: + round_num += 1 + next_paths = [] + for batch_start in range(0, len(current_paths), self.BATCH_SIZE): + batch = current_paths[batch_start:batch_start + self.BATCH_SIZE] + if len(batch) == 1: + next_paths.append(batch[0]) + continue + + batch_out = os.path.join( + tmp_dir, f'audio_reduce_r{round_num}_b{batch_start}.m4a' + ) + inputs = [] + for bp in batch: + inputs.extend(['-i', bp]) + + labels = ''.join(f'[{j}:a]' for j in range(len(batch))) + amix_filter = ( + f'{labels}amix=inputs={len(batch)}' + f':duration=longest:normalize=0' + ) + + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-filter_complex', amix_filter, + '-c:a', 'aac', '-b:a', '128k', + '-ar', '44100', '-ac', '2', + batch_out, + ], + description=f'amix reduce round {round_num}, {len(batch)} tracks', + ) + next_paths.append(batch_out) + + for bp in batch: + try: + os.remove(bp) + except OSError: + pass + + logging.info( + f'Audio reduce round {round_num}: {len(current_paths)} -> ' + f'{len(next_paths)} tracks' + ) + current_paths = next_paths + + mixed_audio_path = current_paths[0] + + # Step 3: Overlay mixed audio onto video + logging.info('Overlaying mixed audio onto video...') + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', video_path, + '-i', mixed_audio_path, + '-filter_complex', + '[0:a][1:a]amix=inputs=2:duration=first:normalize=0[aout]', + '-map', '0:v', '-map', '[aout]', + '-c:v', 'copy', + '-c:a', 'aac', '-b:a', '192k', + output_path, + ], + description='overlay mixed audio onto video', + ) + return output_path diff --git a/mtslinker/ffmpeg.py b/mtslinker/ffmpeg.py new file mode 100644 index 0000000..7afb154 --- /dev/null +++ b/mtslinker/ffmpeg.py @@ -0,0 +1,80 @@ +import logging +import shutil +import subprocess + + +class FFmpegRunner: + """Handles ffmpeg/ffprobe execution and GPU detection.""" + + def __init__(self): + self.nvenc_available = None + self.cuda_overlay_available = None + self._check_tools() + + def _check_tools(self): + for tool in ('ffmpeg', 'ffprobe'): + if not shutil.which(tool): + raise RuntimeError( + f'{tool} is not installed. ' + 'Install it with: apt install ffmpeg / brew install ffmpeg' + ) + + def detect_gpu(self): + if self.nvenc_available is not None: + return + self.nvenc_available = self._has_nvenc() + self.cuda_overlay_available = ( + self._has_cuda_overlay() if self.nvenc_available else False + ) + if self.cuda_overlay_available: + logging.info('CUDA overlay + NVENC detected, using full GPU pipeline') + elif self.nvenc_available: + logging.info('NVENC detected (no CUDA overlay), using GPU encoder only') + else: + logging.info('No GPU support, using CPU pipeline') + + def _has_nvenc(self) -> bool: + try: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-f', 'lavfi', '-i', + 'nullsrc=s=64x64:d=0.1', '-c:v', 'h264_nvenc', '-f', 'null', '-'], + capture_output=True, timeout=10, + ) + return result.returncode == 0 + except Exception: + return False + + def _has_cuda_overlay(self) -> bool: + try: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-filters'], + capture_output=True, text=True, timeout=10, + ) + return 'overlay_cuda' in result.stdout + except Exception: + return False + + def get_video_encoder(self) -> list: + self.detect_gpu() + if self.nvenc_available: + return ['-c:v', 'h264_nvenc', '-preset', 'p4', '-cq', '23'] + return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] + + def get_video_encoder_fast(self) -> list: + self.detect_gpu() + if self.nvenc_available: + return ['-c:v', 'h264_nvenc', '-preset', 'p1', '-cq', '28'] + return ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23'] + + def run(self, cmd: list, description: str = 'ffmpeg'): + logging.debug(f'Running {description}: {" ".join(cmd[:6])}...') + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + stderr = result.stderr.strip() if result.stderr else '(no stderr)' + logging.error(f'{description} failed (exit {result.returncode}):\n{stderr}') + raise subprocess.CalledProcessError( + result.returncode, cmd[0], result.stdout, result.stderr + ) + if result.stderr and result.stderr.strip(): + logging.debug(f'{description} stderr: {result.stderr.strip()[:500]}') + return result diff --git a/mtslinker/grid.py b/mtslinker/grid.py new file mode 100644 index 0000000..9815332 --- /dev/null +++ b/mtslinker/grid.py @@ -0,0 +1,77 @@ +import math +import os + +from mtslinker.ffmpeg import FFmpegRunner + + +class GridCompositor: + """Composites multiple webcam streams into a grid layout.""" + + def __init__(self, ffmpeg: FFmpegRunner): + self.ffmpeg = ffmpeg + + def compute_layout(self, n: int): + cols = math.ceil(math.sqrt(n)) + rows = math.ceil(n / cols) + return cols, rows + + def even(self, x: int) -> int: + return x & ~1 + + def composite(self, active_segments: list, duration: float, + output_path: str, target_w: int, target_h: int) -> str: + n = len(active_segments) + cols, rows = self.compute_layout(n) + cell_w = self.even(target_w // cols) + cell_h = self.even(target_h // rows) + + inputs = [] + filter_parts = [] + labels = [] + + for i, (path, offset) in enumerate(active_segments): + inputs.extend(['-ss', str(offset), '-i', path]) + label = f'v{i}' + filter_parts.append( + f'[{i}:v]scale=w=min({cell_w}\\,iw):h=min({cell_h}\\,ih)' + f':force_original_aspect_ratio=decrease,' + f'pad={cell_w}:{cell_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[{label}]' + ) + labels.append(f'[{label}]') + + total_cells = cols * rows + for i in range(n, total_cells): + idx = len(active_segments) + i - n + inputs.extend(['-f', 'lavfi', '-i', + f'color=black:s={cell_w}x{cell_h}:d={duration}:r=25']) + labels.append(f'[{idx}:v]') + + layout_parts = [] + for i in range(total_cells): + c = i % cols + r = i // cols + layout_parts.append(f'{c * cell_w}_{r * cell_h}') + layout = '|'.join(layout_parts) + + filter_graph = ';'.join(filter_parts) + if filter_graph: + filter_graph += ';' + filter_graph += ( + ''.join(labels) + + f'xstack=inputs={total_cells}:layout={layout}[out]' + ) + + cmd = [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-t', str(duration), + '-filter_complex', filter_graph, + '-map', '[out]', '-map', '0:a?', + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + output_path, + ] + self.ffmpeg.run(cmd, description=f'grid {n} webcams, {duration:.0f}s') + return output_path diff --git a/mtslinker/prober.py b/mtslinker/prober.py new file mode 100644 index 0000000..da81746 --- /dev/null +++ b/mtslinker/prober.py @@ -0,0 +1,179 @@ +import json +import logging +import os +import subprocess +import tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Tuple + + +class MediaProber: + """Probes media files for stream info, duration, resolution, audio levels.""" + + def probe_streams(self, file_path: str) -> dict: + result = subprocess.run( + ['ffprobe', '-v', 'quiet', '-print_format', 'json', + '-show_streams', '-show_format', file_path], + capture_output=True, text=True, + ) + if result.returncode != 0: + return {} + return json.loads(result.stdout) + + def is_valid(self, file_path: str) -> bool: + result = subprocess.run( + ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', + '-of', 'csv=p=0', file_path], + capture_output=True, text=True, + ) + if result.returncode != 0: + logging.warning(f'Corrupt/invalid file: {file_path}: {result.stderr.strip()[:200]}') + return False + return True + + def has_video(self, file_path: str) -> bool: + info = self.probe_streams(file_path) + return any(s.get('codec_type') == 'video' for s in info.get('streams', [])) + + def has_audio(self, file_path: str) -> bool: + info = self.probe_streams(file_path) + return any(s.get('codec_type') == 'audio' for s in info.get('streams', [])) + + def get_duration(self, file_path: str) -> float: + info = self.probe_streams(file_path) + fmt = info.get('format', {}) + if 'duration' in fmt: + return float(fmt['duration']) + for stream in info.get('streams', []): + if 'duration' in stream: + return float(stream['duration']) + return 0.0 + + def get_video_params(self, file_path: str) -> Tuple[int, int, str]: + info = self.probe_streams(file_path) + for stream in info.get('streams', []): + if stream.get('codec_type') == 'video': + w = int(stream.get('width', 1920)) + h = int(stream.get('height', 1080)) + pix_fmt = stream.get('pix_fmt', 'yuv420p') + return w, h, pix_fmt + return 1920, 1080, 'yuv420p' + + def is_silent(self, file_path: str, threshold: float = -88.0) -> bool: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-i', file_path, + '-t', '10', '-af', 'volumedetect', '-f', 'null', '-'], + capture_output=True, text=True, + ) + for line in result.stderr.splitlines(): + if 'mean_volume' in line: + try: + vol = float(line.split('mean_volume:')[1].strip().split()[0]) + return vol < threshold + except (ValueError, IndexError): + pass + return True + + def analyze_audio_levels(self, file_path: str, window_sec: float = 2.0, + sample_rate: int = 44100) -> List[Tuple[float, float]]: + reset_samples = int(sample_rate * window_sec) + with tempfile.NamedTemporaryFile(suffix='.txt', delete=False) as tmp: + tmp_path = tmp.name + try: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-i', file_path, + '-af', f'astats=metadata=1:reset={reset_samples},' + f'ametadata=print:key=lavfi.astats.Overall.RMS_level' + f':file={tmp_path}', + '-f', 'null', '-'], + capture_output=True, text=True, + ) + if result.returncode != 0: + return [] + levels = [] + current_time = None + with open(tmp_path) as f: + for line in f: + line = line.strip() + if line.startswith('frame:'): + for part in line.split(): + if part.startswith('pts_time:'): + try: + current_time = float(part.split(':')[1]) + except (ValueError, IndexError): + pass + elif 'RMS_level' in line and current_time is not None: + try: + val = float(line.split('=')[1]) + levels.append((current_time, val)) + except (ValueError, IndexError): + pass + return levels + finally: + try: + os.remove(tmp_path) + except OSError: + pass + + def probe_file(self, file_path: str) -> dict: + info = self.probe_streams(file_path) + streams = info.get('streams', []) + fmt = info.get('format', {}) + + has_video = any(s.get('codec_type') == 'video' for s in streams) + has_audio = any(s.get('codec_type') == 'audio' for s in streams) + + width, height, pix_fmt = 0, 0, 'yuv420p' + for s in streams: + if s.get('codec_type') == 'video': + width = int(s.get('width', 0)) + height = int(s.get('height', 0)) + pix_fmt = s.get('pix_fmt', 'yuv420p') + break + + duration = 0.0 + if 'duration' in fmt: + duration = float(fmt['duration']) + else: + for s in streams: + if 'duration' in s: + duration = float(s['duration']) + break + + valid = bool(streams) and duration > 0 + return { + 'path': file_path, + 'valid': valid, + 'has_video': has_video, + 'has_audio': has_audio, + 'width': width, + 'height': height, + 'pix_fmt': pix_fmt, + 'duration': duration, + } + + def probe_all_files(self, downloaded_files: list) -> List[dict]: + results = [] + items = [ + (item[0], item[1], + item[2] if len(item) > 2 else None, + item[3] if len(item) > 3 else False, + item[4] if len(item) > 4 else 0) + for item in downloaded_files + ] + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {} + for path, start_time, conf_id, is_admin, api_duration in items: + fut = pool.submit(self.probe_file, path) + futures[fut] = (start_time, conf_id, is_admin, api_duration) + for fut in as_completed(futures): + start_time, conf_id, is_admin, api_duration = futures[fut] + info = fut.result() + info['start_time'] = start_time + info['conf_id'] = conf_id + info['is_admin'] = is_admin + if api_duration > 0: + info['api_duration'] = api_duration + results.append(info) + results.sort(key=lambda x: x['start_time']) + return results diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 5fb37ce..e01df25 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1,793 +1,533 @@ +"""Video processor — two-phase pipeline: analyze then execute. + +Uses composition of specialized classes: + FFmpegRunner — ffmpeg execution, GPU detection + MediaProber — file probing, duration, streams + SegmentBuilder — normalize, gaps, dedup + GridCompositor — multi-webcam grid layout + SlideCompositor — presentation slide overlay + AudioMerger — batched audio mixing +""" import json import logging -import math import os import shutil import subprocess +from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Dict, List, Tuple, Union -AUDIO_MERGE_BATCH_SIZE = 8 - - -def _has_nvenc() -> bool: - """Check if NVIDIA NVENC hardware encoder is available.""" - try: - result = subprocess.run( - ['ffmpeg', '-v', 'error', '-f', 'lavfi', '-i', 'nullsrc=s=64x64:d=0.1', - '-c:v', 'h264_nvenc', '-f', 'null', '-'], - capture_output=True, timeout=10, - ) - return result.returncode == 0 - except Exception: - return False - - -def _has_cuda_overlay() -> bool: - """Check if ffmpeg has CUDA overlay filter support.""" - try: - result = subprocess.run( - ['ffmpeg', '-v', 'error', '-filters'], - capture_output=True, text=True, timeout=10, - ) - return 'overlay_cuda' in result.stdout - except Exception: - return False - - -# Detected once at import time -_NVENC_AVAILABLE = None -_CUDA_OVERLAY_AVAILABLE = None - - -def _detect_gpu(): - """Detect GPU capabilities once.""" - global _NVENC_AVAILABLE, _CUDA_OVERLAY_AVAILABLE - if _NVENC_AVAILABLE is None: - _NVENC_AVAILABLE = _has_nvenc() - _CUDA_OVERLAY_AVAILABLE = _has_cuda_overlay() if _NVENC_AVAILABLE else False - if _CUDA_OVERLAY_AVAILABLE: - logging.info('CUDA overlay + NVENC detected, using full GPU pipeline') - elif _NVENC_AVAILABLE: - logging.info('NVENC detected (no CUDA overlay), using GPU encoder only') - else: - logging.info('No GPU support, using CPU pipeline') - +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.segments import SegmentBuilder +from mtslinker.grid import GridCompositor +from mtslinker.slides import SlideCompositor +from mtslinker.audio import AudioMerger + + +class VideoProcessor: + """Orchestrates the two-phase video processing pipeline.""" + + def __init__(self): + self.ffmpeg = FFmpegRunner() + self.prober = MediaProber() + self.segments = SegmentBuilder(self.ffmpeg, self.prober) + self.grid = GridCompositor(self.ffmpeg) + self.slides = SlideCompositor(self.ffmpeg) + self.audio = AudioMerger(self.ffmpeg, self.prober) + + def analyze(self, total_duration, downloaded_files, directory, + output_path, max_duration=None, slide_events=None): + """Phase 1: Probe all files, make all decisions, write manifest.""" + logging.info('Phase 1: Analyzing files...') + + all_files = self.prober.probe_all_files(downloaded_files) + + video_files = [] + audio_files = [] + skipped = 0 + errors = [] + + for f in all_files: + if not f['valid']: + skipped += 1 + errors.append(f'Corrupt/invalid: {f["path"]}') + continue + if f['has_video']: + video_files.append(f) + else: + audio_files.append(f) -def _get_video_encoder() -> list: - """Return ffmpeg video encoder args, preferring NVENC if available.""" - _detect_gpu() - if _NVENC_AVAILABLE: - return ['-c:v', 'h264_nvenc', '-preset', 'p4', '-cq', '23'] - return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] + if skipped: + logging.warning(f'Skipped {skipped} corrupt/invalid files') + logging.info(f'Segments: {len(video_files)} video, {len(audio_files)} audio-only') + if not video_files: + raise ValueError('No valid video segments found.') -def _get_video_encoder_fast() -> list: - """Return fast encoder args for simple content (slides, gaps).""" - _detect_gpu() - if _NVENC_AVAILABLE: - return ['-c:v', 'h264_nvenc', '-preset', 'p1', '-cq', '28'] - return ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23'] + # Determine target resolution + target_w, target_h, target_pix_fmt = 0, 0, 'yuv420p' + for f in video_files: + if f['width'] * f['height'] > target_w * target_h: + target_w, target_h, target_pix_fmt = f['width'], f['height'], f['pix_fmt'] + if target_w * target_h < 640 * 360: + target_w, target_h = 640, 360 + MAX_H = 720 + if target_h > MAX_H: + target_w = int(target_w * MAX_H / target_h) + target_w -= target_w % 2 + target_h = MAX_H + logging.info(f'Target resolution: {target_w}x{target_h}, pix_fmt={target_pix_fmt}') + + # Detect overlaps + has_overlaps = False + for i in range(1, len(video_files)): + prev_dur = video_files[i-1].get('api_duration', 0) or video_files[i-1]['duration'] + prev_end = video_files[i-1]['start_time'] + prev_dur + if video_files[i]['start_time'] < prev_end - 0.5: + has_overlaps = True + break + # Decide strategy and build segment plan + overlap_strategy = 'none' + kept_video = [] + extra_audio = [] + segments = [] -def _check_ffmpeg(): - """Check that ffmpeg and ffprobe are available.""" - for tool in ('ffmpeg', 'ffprobe'): - if not shutil.which(tool): - raise RuntimeError( - f'{tool} is not installed. ' - 'Install it with: apt install ffmpeg / brew install ffmpeg' - ) + if has_overlaps and slide_events: + overlap_strategy = 'dedup' + vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), + f.get('is_admin', False)) for f in video_files] + deduped = self.segments.deduplicate(vf_tuples) + kept_paths = {v[0] for v in deduped} + + dedup_map = {f['path']: f for f in video_files} + for path, start_time in deduped: + if path in dedup_map: + kept_video.append(dedup_map[path]) + + for f in video_files: + if f['path'] not in kept_paths and f['has_audio']: + extra_audio.append(f) + + logging.info(f'Dedup: kept {len(kept_video)}, ' + f'{len(extra_audio)} extra audio from webcams') + + elif has_overlaps: + overlap_strategy = 'grid' + annotated = [] + for f in video_files: + dur = f.get('api_duration', 0) or f['duration'] + annotated.append((f['path'], f['start_time'], dur, + f['start_time'] + dur, f['has_audio'])) + + event_times_set = set() + for _, start, _, end, _ in annotated: + event_times_set.add(start) + event_times_set.add(end) + event_times_set.add(total_duration) + event_times = sorted(event_times_set) + + grid_windows = [] + prev_active_key = None + window_start = None + window_active = None + for ei in range(len(event_times) - 1): + t_start = event_times[ei] + t_end = event_times[ei + 1] + if t_end - t_start < 0.1: + continue + active = [] + for path, seg_start, dur, seg_end, has_audio in annotated: + if seg_start < t_end and seg_end > t_start: + offset = max(0, t_start - seg_start) + remaining = dur - offset + if remaining > 0.5: + active.append({'path': path, 'offset': offset, + 'remaining': remaining, + 'has_audio': has_audio}) + active_key = tuple(a['path'] for a in active) + if active_key == prev_active_key and window_start is not None: + continue + if prev_active_key is not None and window_start is not None: + grid_windows.append({ + 'start_time': window_start, + 'duration': t_start - window_start, + 'sources': window_active, + }) + window_start = t_start + prev_active_key = active_key + window_active = active + if window_start is not None and window_active: + grid_windows.append({ + 'start_time': window_start, + 'duration': event_times[-1] - window_start, + 'sources': window_active, + }) + for gw in grid_windows: + if not gw['sources']: + segments.append({ + 'type': 'gap', + 'duration': gw['duration'], + 'start_time': gw['start_time'], + }) + else: + segments.append({ + 'type': 'grid', + 'start_time': gw['start_time'], + 'planned_duration': gw['duration'], + 'sources': gw['sources'], + }) + + # Extract audio from ALL video files with audio for mixing + for f in video_files: + if f['has_audio']: + extra_audio.append(f) + if extra_audio: + logging.info(f'Grid: {len(extra_audio)} webcam audio tracks to extract') -def _is_valid_media(file_path: str) -> bool: - """Check if a media file is valid (not corrupt / has moov atom).""" - result = subprocess.run( - ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', - '-of', 'csv=p=0', file_path], - capture_output=True, text=True, - ) - if result.returncode != 0: - logging.warning(f'Corrupt/invalid file: {file_path}: {result.stderr.strip()[:200]}') - return False - return True - - -def _ffprobe_streams(file_path: str) -> dict: - """Return ffprobe stream info for a file.""" - result = subprocess.run( - [ - 'ffprobe', '-v', 'quiet', - '-print_format', 'json', - '-show_streams', - '-show_format', - file_path, - ], - capture_output=True, text=True, - ) - if result.returncode != 0: - return {} - return json.loads(result.stdout) - - -def _is_silent(file_path: str, threshold: float = -88.0) -> bool: - """Check if an audio file is effectively silent (mean volume below threshold).""" - result = subprocess.run( - ['ffmpeg', '-v', 'error', '-i', file_path, - '-t', '10', '-af', 'volumedetect', '-f', 'null', '-'], - capture_output=True, text=True, - ) - for line in result.stderr.splitlines(): - if 'mean_volume' in line: - try: - vol = float(line.split('mean_volume:')[1].strip().split()[0]) - return vol < threshold - except (ValueError, IndexError): - pass - return True # if we can't detect, treat as silent - - -def _analyze_audio_levels(file_path: str, window_sec: float = 2.0, - sample_rate: int = 44100) -> List[Tuple[float, float]]: - """Analyze RMS audio levels per time window. - - Returns [(time_offset_in_file, rms_db), ...] for each window. - """ - import tempfile - reset_samples = int(sample_rate * window_sec) - with tempfile.NamedTemporaryFile(suffix='.txt', delete=False) as tmp: - tmp_path = tmp.name - - try: - result = subprocess.run( - ['ffmpeg', '-v', 'error', '-i', file_path, - '-af', f'astats=metadata=1:reset={reset_samples},' - f'ametadata=print:key=lavfi.astats.Overall.RMS_level' - f':file={tmp_path}', - '-f', 'null', '-'], - capture_output=True, text=True, - ) - if result.returncode != 0: - return [] - - levels = [] - current_time = None - with open(tmp_path) as f: - for line in f: - line = line.strip() - if line.startswith('frame:'): - # Extract pts_time from "frame:N pts:N pts_time:N.NNN" - for part in line.split(): - if part.startswith('pts_time:'): - try: - current_time = float(part.split(':')[1]) - except (ValueError, IndexError): - pass - elif 'RMS_level' in line and current_time is not None: - try: - val = float(line.split('=')[1]) - levels.append((current_time, val)) - except (ValueError, IndexError): - pass - return levels - finally: - try: - os.remove(tmp_path) - except OSError: - pass - - -def _build_speaker_timeline( - video_files: list, - total_duration: float, - window_sec: float = 2.0, - silence_threshold: float = -50.0, - min_hold_sec: float = 4.0, -) -> List[Tuple[float, float, str]]: - """Build a timeline of which conf_id should be shown at each moment. - - Picks the loudest non-admin speaker when above silence_threshold, - otherwise defaults to admin. Applies hysteresis to prevent flickering. - - Returns [(interval_start, interval_end, conf_id), ...]. - """ - from collections import defaultdict - - if not video_files: - return [] - - # Group by conf_id, identify admin conf - conf_segments = defaultdict(list) # conf_id -> [(path, start_time)] - admin_conf = None - conf_is_admin = {} - - for item in video_files: - path, start = item[0], item[1] - conf_id = item[2] if len(item) > 2 else None - is_admin = item[3] if len(item) > 3 else False - if conf_id is None: - continue - conf_segments[conf_id].append((path, start)) - if is_admin: - conf_is_admin[conf_id] = True - - if not conf_segments: - return [] - - # Find admin conf (most duration among admin confs, or most duration overall) - conf_total_dur = {} - for conf_id, segs in conf_segments.items(): - conf_total_dur[conf_id] = sum(_get_duration(p) for p, _ in segs) - - admin_confs = {c for c in conf_segments if conf_is_admin.get(c)} - if admin_confs: - admin_conf = max(admin_confs, key=lambda c: conf_total_dur.get(c, 0)) - elif conf_total_dur: - admin_conf = max(conf_total_dur, key=conf_total_dur.get) - - # Analyze audio levels for each segment and map to absolute timeline - # conf_id -> {window_index: max_rms} - n_windows = int(total_duration / window_sec) + 1 - conf_levels = defaultdict(lambda: defaultdict(lambda: -91.0)) - - logging.info(f'Speaker detection: analyzing audio for {len(conf_segments)} participants...') - - # Build list of (conf_id, path, seg_start) tasks for parallel analysis - analysis_tasks = [] - for conf_id, segs in conf_segments.items(): - for path, seg_start in segs: - analysis_tasks.append((conf_id, path, seg_start)) - - def _analyze_task(task): - conf_id, path, seg_start = task - levels = _analyze_audio_levels(path, window_sec) - return conf_id, seg_start, levels - - with ThreadPoolExecutor(max_workers=4) as pool: - futures = {pool.submit(_analyze_task, t): t for t in analysis_tasks} - for fut in as_completed(futures): - conf_id, seg_start, levels = fut.result() - for time_offset, rms in levels: - abs_time = seg_start + time_offset - win_idx = int(abs_time / window_sec) - if 0 <= win_idx < n_windows: - if rms > conf_levels[conf_id][win_idx]: - conf_levels[conf_id][win_idx] = rms - - # For each window, pick the speaker - all_confs = list(conf_segments.keys()) - non_admin = [c for c in all_confs if c != admin_conf] - raw_picks = [] - - for win_idx in range(n_windows): - best_non_admin = None - best_rms = silence_threshold - for conf_id in non_admin: - rms = conf_levels[conf_id][win_idx] - if rms > best_rms: - best_rms = rms - best_non_admin = conf_id - raw_picks.append(best_non_admin if best_non_admin else admin_conf) - - # Apply hysteresis: revert short non-admin bursts to admin - min_hold_windows = max(1, int(min_hold_sec / window_sec)) - picks = list(raw_picks) - i = 0 - while i < len(picks): - if picks[i] != admin_conf: - # Find run length of this non-admin speaker - j = i - while j < len(picks) and picks[j] == picks[i]: - j += 1 - if j - i < min_hold_windows: - # Too short, revert to admin - for k in range(i, j): - picks[k] = admin_conf - i = j else: - i += 1 - - # Merge consecutive same-speaker windows into intervals - intervals = [] - if picks: - current_conf = picks[0] - interval_start = 0.0 - for win_idx in range(1, len(picks)): - if picks[win_idx] != current_conf: - interval_end = win_idx * window_sec - intervals.append((interval_start, interval_end, current_conf)) - current_conf = picks[win_idx] - interval_start = interval_end - # Final interval - intervals.append((interval_start, total_duration, current_conf)) - - # Log summary - non_admin_time = sum( - end - start for start, end, conf in intervals if conf != admin_conf - ) - logging.info( - f'Speaker timeline: {len(intervals)} intervals, ' - f'{non_admin_time:.0f}s non-admin out of {total_duration:.0f}s' - ) - - return intervals - - -def _build_speaker_switched_segments( - video_files: list, - speaker_timeline: List[Tuple[float, float, str]], - tmp_dir: str, - target_w: int, - target_h: int, -) -> List[Tuple[str, float]]: - """Cut webcam segments according to the speaker timeline. - - For each interval, extracts the appropriate slice from the source - webcam segment for that conf_id. Returns [(path, start_time), ...] - in the same format as _deduplicate_overlapping. - """ - if not speaker_timeline: - return [] - - # Build lookup: conf_id -> sorted [(path, start, duration, end)] - from collections import defaultdict - conf_segs = defaultdict(list) - for item in video_files: - path, start = item[0], item[1] - conf_id = item[2] if len(item) > 2 else None - dur = _get_duration(path) - conf_segs[conf_id].append((path, start, dur, start + dur)) - for conf_id in conf_segs: - conf_segs[conf_id].sort(key=lambda x: x[1]) - - speaker_dir = os.path.join(tmp_dir, 'speaker_segments') - os.makedirs(speaker_dir, exist_ok=True) - - result = [] - for i, (t_start, t_end, conf_id) in enumerate(speaker_timeline): - duration = t_end - t_start - if duration < 0.1: - continue + vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), + f.get('is_admin', False)) for f in video_files] + deduped = self.segments.deduplicate(vf_tuples) + dedup_map = {f['path']: f for f in video_files} + for path, start_time in deduped: + if path in dedup_map: + kept_video.append(dedup_map[path]) + + # Build segment plan for non-grid strategies + if overlap_strategy != 'grid': + segments = [] + current_time = 0.0 + for i, f in enumerate(kept_video): + start_time = f['start_time'] + if start_time < current_time - 0.5: + errors.append(f'Segment {i} overlaps at {start_time:.1f}s (current={current_time:.1f}s)') + continue - # Find the source segment covering this interval - source = None - for path, seg_start, seg_dur, seg_end in conf_segs.get(conf_id, []): - if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: - source = (path, seg_start) - break + gap = start_time - current_time + if gap > 0.1: + segments.append({ + 'type': 'gap', 'duration': gap, 'start_time': current_time, + }) - if source is None: - # No segment for this conf_id at this time — try any conf - for cid, segs in conf_segs.items(): - for path, seg_start, seg_dur, seg_end in segs: - if seg_start <= t_start + 0.5 and seg_end >= t_start + 0.5: - source = (path, seg_start) - break - if source: + max_dur = 0 + for j in range(i + 1, len(kept_video)): + ns = kept_video[j]['start_time'] + if ns > start_time + 0.5: + max_dur = ns - start_time break - if source is None: - continue - - src_path, seg_start = source - offset = t_start - seg_start - seg_path = os.path.join(speaker_dir, f'speaker_{i:04d}.mp4') - - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(max(0, offset)), - '-i', src_path, - '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'speaker segment {i+1}/{len(speaker_timeline)} ' - f'({duration:.0f}s, conf {conf_id})', - ) - result.append((seg_path, t_start)) - - logging.info(f'Speaker switching: built {len(result)} segments') - return result - - -def _has_video_stream(file_path: str) -> bool: - """Check if a file contains a video stream.""" - info = _ffprobe_streams(file_path) - for stream in info.get('streams', []): - if stream.get('codec_type') == 'video': - return True - return False - - -def _get_duration(file_path: str) -> float: - """Get duration of a media file in seconds.""" - info = _ffprobe_streams(file_path) - fmt = info.get('format', {}) - if 'duration' in fmt: - return float(fmt['duration']) - for stream in info.get('streams', []): - if 'duration' in stream: - return float(stream['duration']) - return 0.0 - - -def _get_video_params(file_path: str) -> Tuple[int, int, str]: - """Get width, height, and pixel format of the first video stream.""" - info = _ffprobe_streams(file_path) - for stream in info.get('streams', []): - if stream.get('codec_type') == 'video': - w = int(stream.get('width', 1920)) - h = int(stream.get('height', 1080)) - pix_fmt = stream.get('pix_fmt', 'yuv420p') - return w, h, pix_fmt - return 1920, 1080, 'yuv420p' - - -def _run_ffmpeg(cmd: list, description: str = 'ffmpeg'): - """Run an ffmpeg command, logging stderr on failure.""" - logging.debug(f'Running {description}: {" ".join(cmd[:6])}...') - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - stderr = result.stderr.strip() if result.stderr else '(no stderr)' - logging.error(f'{description} failed (exit {result.returncode}):\n{stderr}') - raise subprocess.CalledProcessError( - result.returncode, cmd[0], result.stdout, result.stderr - ) - if result.stderr and result.stderr.strip(): - logging.debug(f'{description} stderr: {result.stderr.strip()[:500]}') - return result - - -def _generate_black_segment(output_path: str, duration: float, - width: int = 1920, height: int = 1080, - pix_fmt: str = 'yuv420p') -> str: - """Generate a short black video+silent audio segment with ffmpeg.""" - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-f', 'lavfi', '-i', f'color=c=black:s={width}x{height}:d={duration}:r=25', - '-f', 'lavfi', '-i', f'anullsrc=r=44100:cl=stereo', - '-t', str(duration), - *_get_video_encoder_fast(), - '-pix_fmt', pix_fmt, - '-c:a', 'aac', '-b:a', '128k', - '-shortest', - output_path, - ], - description=f'generate black segment ({duration:.1f}s)', - ) - return output_path - - -def _ensure_audio_stream(input_path: str, output_path: str) -> str: - """If the file has no audio stream, add a silent one so concat works.""" - info = _ffprobe_streams(input_path) - has_audio = any( - s.get('codec_type') == 'audio' for s in info.get('streams', []) - ) - if has_audio: - return input_path - - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-i', input_path, - '-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo', - '-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k', - '-shortest', - output_path, - ], - description='add silent audio stream', - ) - return output_path - - -def _normalize_segment(input_path: str, output_path: str, - width: int, height: int, pix_fmt: str, - max_duration: float = 0) -> str: - """Re-encode a segment to a common format for reliable concatenation. - - Args: - max_duration: If > 0, truncate the output to this many seconds. - """ - duration_args = ['-t', str(max_duration)] if max_duration > 0 else [] - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-i', input_path, - '-vf', f'scale={width}:{height}:force_original_aspect_ratio=decrease,' - f'pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black,' - f'setsar=1', - '-pix_fmt', pix_fmt, - *_get_video_encoder_fast(), - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - *duration_args, - output_path, - ], - description=f'normalize segment {os.path.basename(input_path)}', - ) - return output_path - - -def _compute_grid(n: int) -> Tuple[int, int]: - """Return (cols, rows) for a grid holding n items.""" - cols = math.ceil(math.sqrt(n)) - rows = math.ceil(n / cols) - return cols, rows - - -def _even(x: int) -> int: - """Round down to nearest even number (ffmpeg requires even dimensions).""" - return x & ~1 - - -def _composite_grid( - active_segments: list, - duration: float, - output_path: str, - target_w: int, - target_h: int, -) -> str: - """Create a grid video from multiple simultaneous webcam segments. - - Args: - active_segments: list of (path, offset_within_file) tuples - duration: length of this time window - output_path: where to write - target_w, target_h: output resolution - """ - n = len(active_segments) - cols, rows = _compute_grid(n) - cell_w = _even(target_w // cols) - cell_h = _even(target_h // rows) - - inputs = [] - filter_parts = [] - labels = [] - - for i, (path, offset) in enumerate(active_segments): - inputs.extend(['-ss', str(offset), '-i', path]) - label = f'v{i}' - # Scale to fit cell, then force exact cell dimensions with pad - filter_parts.append( - f'[{i}:v]scale=w=min({cell_w}\\,iw):h=min({cell_h}\\,ih)' - f':force_original_aspect_ratio=decrease,' - f'pad={cell_w}:{cell_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[{label}]' - ) - labels.append(f'[{label}]') - - # Pad with black cells if needed - total_cells = cols * rows - for i in range(n, total_cells): - idx = len(active_segments) + i - n - inputs.extend(['-f', 'lavfi', '-i', - f'color=black:s={cell_w}x{cell_h}:d={duration}:r=25']) - labels.append(f'[{idx}:v]') - - # Build xstack layout string: x_y positions - layout_parts = [] - for i in range(total_cells): - c = i % cols - r = i // cols - layout_parts.append(f'{c * cell_w}_{r * cell_h}') - layout = '|'.join(layout_parts) - - filter_graph = ';'.join(filter_parts) - if filter_graph: - filter_graph += ';' - filter_graph += ( - ''.join(labels) - + f'xstack=inputs={total_cells}:layout={layout}[out]' - ) - - # Include audio from first input if available (? = optional) - cmd = [ - 'ffmpeg', '-y', '-v', 'error', - *inputs, - '-t', str(duration), - '-filter_complex', filter_graph, - '-map', '[out]', '-map', '0:a?', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - output_path, - ] - _run_ffmpeg(cmd, description=f'grid {n} webcams, {duration:.0f}s') - return output_path - - -def _build_grid_segments( - video_files: list, - tmp_dir: str, - target_w: int, - target_h: int, - total_duration: float, -) -> List[Tuple[str, float]]: - """Build grid-composited segments from overlapping webcam streams. - - Instead of deduplicating, composites all concurrent webcams into a grid. - Returns a list of (segment_path, start_time) ready for concatenation. - """ - # Annotate with duration - annotated = [] - for item in video_files: - path, start = item[0], item[1] - dur = _get_duration(path) - if dur > 0: - annotated.append((path, start, dur, start + dur)) - - if not annotated: - return [] - - # Collect all event times (segment starts and ends) - events = set() - for _, start, _, end in annotated: - events.add(start) - events.add(end) - events.add(total_duration) - event_times = sorted(events) - - # Merge adjacent windows with the same active set - grid_dir = os.path.join(tmp_dir, 'grid_segments') - os.makedirs(grid_dir, exist_ok=True) - - result = [] - prev_active = None - window_start = None - - for i in range(len(event_times) - 1): - t_start = event_times[i] - t_end = event_times[i + 1] - if t_end - t_start < 0.1: - continue - - # Find active segments in this window - active = [] - for path, seg_start, dur, seg_end in annotated: - if seg_start < t_end and seg_end > t_start: - offset = max(0, t_start - seg_start) - active.append((path, offset)) - - active_key = tuple(a[0] for a in active) - - # Merge with previous window if same active set - if active_key == prev_active and window_start is not None: - continue # will be handled when active set changes - - # Emit previous merged window - if prev_active is not None and window_start is not None: - merged_end = t_start - merged_dur = merged_end - window_start - if merged_dur > 0.1: - _emit_grid_window( - prev_active_segs, merged_dur, window_start, - grid_dir, len(result), target_w, target_h, result, - ) - - window_start = t_start - prev_active = active_key - prev_active_segs = active - - # Emit final window - if prev_active is not None and window_start is not None: - merged_end = event_times[-1] - merged_dur = merged_end - window_start - if merged_dur > 0.1: - _emit_grid_window( - prev_active_segs, merged_dur, window_start, - grid_dir, len(result), target_w, target_h, result, - ) - - logging.info(f'Grid: built {len(result)} segments from ' - f'{len(annotated)} webcam streams') - return result - - -def _emit_grid_window(active, duration, start_time, grid_dir, idx, - target_w, target_h, result): - """Helper to emit a single grid window segment.""" - if not active: - return - seg_path = os.path.join(grid_dir, f'grid_{idx}.mp4') - if len(active) == 1: - # Single webcam — just extract the slice - path, offset = active[0] - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-ss', str(offset), '-i', path, - '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', - seg_path, - ], - description=f'grid window {idx} (1 webcam, {duration:.0f}s)', - ) - else: - _composite_grid(active, duration, seg_path, target_w, target_h) - result.append((seg_path, start_time)) - + file_dur = f.get('api_duration', 0) or f['duration'] + planned_dur = max_dur if max_dur > 0 else file_dur + segments.append({ + 'type': 'video', + 'source_path': f['path'], + 'start_time': start_time, + 'source_duration': f['duration'], + 'max_duration': max_dur, + 'planned_duration': planned_dur, + 'has_audio': f['has_audio'], + 'width': f['width'], + 'height': f['height'], + }) + current_time = start_time + planned_dur + + # Fill grid timeline gaps + if overlap_strategy == 'grid' and segments: + filled = [] + current_time = 0.0 + for s in segments: + start = s.get('start_time', 0) + if start - current_time > 0.1: + filled.append({ + 'type': 'gap', + 'duration': start - current_time, + 'start_time': current_time, + }) + filled.append(s) + current_time = start + s.get('planned_duration', s.get('duration', 0)) + segments = filled + + # Trailing gap + if overlap_strategy == 'grid': + current_time = 0 + for s in segments: + end = s.get('start_time', 0) + s.get('planned_duration', s.get('duration', 0)) + if end > current_time: + current_time = end + if current_time < total_duration - 0.1: + segments.append({ + 'type': 'gap', + 'duration': total_duration - current_time, + 'start_time': current_time, + }) -def _extract_admin_conf_ids(json_data: Dict) -> set: - """Find conference IDs belonging to ADMIN users. + # Audio tracks + all_audio = [{'path': f['path'], 'start_time': f['start_time'], + 'origin': 'original'} for f in audio_files] + for f in extra_audio: + all_audio.append({ + 'path': f['path'], + 'start_time': f['start_time'], + 'origin': 'webcam_extract', + }) - Uses userlist events to find ADMIN user IDs, then maps them to - conference IDs via conference.add events. - """ - admin_user_ids = set() - user_to_conf = {} # user_id -> set of conf_ids + # Slide compositing plan + slide_plan = None + if slide_events: + CHUNK_SECS = 1800 + n_chunks = max(1, int(total_duration + CHUNK_SECS - 1) // CHUNK_SECS) + slide_plan = { + 'events': slide_events, + 'chunk_seconds': CHUNK_SECS, + 'num_chunks': n_chunks, + } - for event in json_data.get('eventLogs', []): - if not isinstance(event, dict): - continue - module = event.get('module', '') - data_list = event.get('data', []) - if isinstance(data_list, dict): - data_list = [data_list] - if not isinstance(data_list, list): - continue + # GPU detection + self.ffmpeg.detect_gpu() + gpu = { + 'nvenc': bool(self.ffmpeg.nvenc_available), + 'cuda_overlay': bool(self.ffmpeg.cuda_overlay_available), + } - for d in data_list: - if not isinstance(d, dict): + # Validation warnings + warnings = [] + for seg in segments: + if seg['type'] != 'video': continue + src_dur = seg.get('source_duration', 0) + planned = seg.get('planned_duration', 0) + if src_dur and planned and src_dur < planned - 1.0: + warnings.append( + f'Segment at {seg["start_time"]:.0f}s: source={src_dur:.0f}s ' + f'but planned={planned:.0f}s (will pad {planned-src_dur:.0f}s black)' + ) - if 'userlist' in module: - role = d.get('role', '') - user = d.get('user', {}) - if isinstance(user, dict) and role == 'ADMIN': - uid = user.get('id') - if uid: - admin_user_ids.add(uid) - - if module == 'conference.add': - user = d.get('user', {}) - if isinstance(user, dict): - uid = user.get('id') - cid = d.get('id') - if uid and cid: - user_to_conf.setdefault(uid, set()).add(cid) + manifest = { + 'version': 1, + 'total_duration': total_duration, + 'directory': directory, + 'output_path': output_path, + 'max_duration': max_duration, + 'target': {'width': target_w, 'height': target_h, 'pix_fmt': target_pix_fmt}, + 'overlap_strategy': overlap_strategy, + 'segments': segments, + 'audio_tracks': all_audio, + 'slide_compositing': slide_plan, + 'gpu': gpu, + 'errors': errors, + 'warnings': warnings, + 'stats': { + 'total_files': len(all_files), + 'video_files': len(video_files), + 'audio_files': len(audio_files), + 'kept_video': len(kept_video), + 'extra_audio': len(extra_audio), + 'skipped': skipped, + 'segments': len(segments), + 'gaps': sum(1 for s in segments if s['type'] == 'gap'), + }, + } - admin_confs = set() - for uid in admin_user_ids: - admin_confs.update(user_to_conf.get(uid, set())) + manifest_path = os.path.join(directory, 'manifest.json') + with open(manifest_path, 'w') as f: + json.dump(manifest, f, indent=2, ensure_ascii=False) + logging.info(f'Manifest written to {manifest_path}') + logging.info(f'Plan: {manifest["stats"]["segments"]} segments ' + f'({manifest["stats"]["gaps"]} gaps), ' + f'{len(all_audio)} audio tracks, ' + f'strategy={overlap_strategy}') + + if errors: + for e in errors: + logging.warning(f'Analyze: {e}') + if warnings: + for w in warnings: + logging.warning(f'Analyze: {w}') + + return manifest + + def execute(self, manifest): + """Phase 2: Execute the plan from the manifest.""" + logging.info('Phase 2: Executing plan...') + + directory = manifest['directory'] + output_path = manifest['output_path'] + total_duration = manifest['total_duration'] + target = manifest['target'] + target_w, target_h = target['width'], target['height'] + target_pix_fmt = target['pix_fmt'] + + tmp_dir = os.path.join(directory, '_tmp_ffmpeg') + os.makedirs(tmp_dir, exist_ok=True) + + # Extract audio from dropped webcam files + audio_files = [] + extra_dir = os.path.join(tmp_dir, 'extra_audio') + os.makedirs(extra_dir, exist_ok=True) + extract_count = 0 + for track in manifest['audio_tracks']: + if track['origin'] == 'webcam_extract': + audio_path = os.path.join(extra_dir, f'extra_{extract_count}.m4a') + try: + self.ffmpeg.run( + ['ffmpeg', '-y', '-v', 'error', + '-i', track['path'], '-vn', + '-c:a', 'aac', '-b:a', '128k', + '-ar', '44100', '-ac', '2', + audio_path], + description=f'extract audio from webcam {extract_count}', + ) + audio_files.append((audio_path, track['start_time'])) + extract_count += 1 + except subprocess.CalledProcessError: + logging.warning(f'Failed to extract audio from {track["path"]}') + else: + audio_files.append((track['path'], track['start_time'])) + if extract_count: + logging.info(f'Extracted audio from {extract_count} webcam files') + + # Process segments + segments = manifest['segments'] + grid_dir = os.path.join(tmp_dir, 'grid_segments') + concat_segments = [] + + for i, seg in enumerate(segments): + if seg['type'] == 'gap': + gap_path = os.path.join(tmp_dir, f'gap_{i}.mp4') + self.segments.generate_black(gap_path, seg['duration'], + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + logging.info(f'Generated {seg["duration"]:.1f}s black gap') + + elif seg['type'] == 'grid': + os.makedirs(grid_dir, exist_ok=True) + seg_path = os.path.join(grid_dir, f'grid_{i}.mp4') + duration = seg['planned_duration'] + sources = sorted(seg['sources'], + key=lambda s: not s.get('has_audio', False)) + active = [(s['path'], s['offset']) for s in sources] + try: + if len(active) == 1: + path, offset = active[0] + self.ffmpeg.run( + ['ffmpeg', '-y', '-v', 'error', + '-ss', str(offset), '-i', path, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:' + f'force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', seg_path], + description=f'grid {i} (1 webcam, {duration:.0f}s)', + ) + else: + self.grid.composite(active, duration, seg_path, target_w, target_h) + with_audio = os.path.join(tmp_dir, f'grida_{i}.mp4') + final_seg = self.segments.ensure_audio(seg_path, with_audio) + concat_segments.append(final_seg) + except subprocess.CalledProcessError: + logging.warning(f'Grid segment {i} failed, using black gap') + gap_path = os.path.join(tmp_dir, f'gridfail_{i}.mp4') + self.segments.generate_black(gap_path, duration, + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + + elif seg['type'] == 'video': + norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') + self.segments.normalize(seg['source_path'], norm_path, + target_w, target_h, target_pix_fmt, + max_duration=seg.get('max_duration', 0)) + with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') + final_seg = self.segments.ensure_audio(norm_path, with_audio_path) + + actual_dur = self.prober.get_duration(final_seg) + planned_dur = seg.get('planned_duration', 0) + if planned_dur > 0 and actual_dur < planned_dur - 1.0: + shortfall = planned_dur - actual_dur + logging.warning( + f'Segment {i} is {actual_dur:.1f}s but planned ' + f'{planned_dur:.1f}s — padding {shortfall:.1f}s' + ) + pad_path = os.path.join(tmp_dir, f'pad_{i}.mp4') + self.segments.generate_black(pad_path, shortfall, + target_w, target_h, target_pix_fmt) + concat_segments.append(final_seg) + concat_segments.append(pad_path) + else: + concat_segments.append(final_seg) - if admin_confs: - logging.info(f'Found {len(admin_confs)} conference(s) from ADMIN users') + # Concatenate + concat_list_path = os.path.join(tmp_dir, 'concat.txt') + with open(concat_list_path, 'w') as f: + for seg in concat_segments: + f.write(f"file '{os.path.abspath(seg)}'\n") - return admin_confs + video_only_path = os.path.join(tmp_dir, 'video_concat.mp4') + logging.info(f'Concatenating {len(concat_segments)} segments...') + cap = manifest.get('max_duration') or total_duration + concat_cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-f', 'concat', '-safe', '0', + '-i', concat_list_path, '-c', 'copy', + ] + if cap: + concat_cmd.extend(['-t', str(cap)]) + concat_cmd.append(video_only_path) + self.ffmpeg.run(concat_cmd, description=f'concat {len(concat_segments)} segments') + + # Composite slides + slide_plan = manifest.get('slide_compositing') + if slide_plan and slide_plan.get('events'): + composited_path = os.path.join(tmp_dir, 'video_composited.mp4') + self.slides.composite( + video_only_path, slide_plan['events'], composited_path, + tmp_dir, total_duration, + ) + video_only_path = composited_path + + # Merge audio + if audio_files: + logging.info(f'Merging {len(audio_files)} audio tracks...') + result_path = self.audio.merge( + video_only_path, audio_files, tmp_dir, output_path, + total_duration, + ) + else: + shutil.move(video_only_path, output_path) + result_path = output_path + + shutil.rmtree(tmp_dir, ignore_errors=True) + logging.info(f'Final video saved to {result_path}') + return result_path + + def process(self, total_duration, downloaded_files, directory, + output_path, max_duration=None, slide_events=None): + """Full pipeline: analyze then execute.""" + manifest = self.analyze( + total_duration, downloaded_files, directory, output_path, + max_duration, slide_events, + ) + return self.execute(manifest) -def process_and_download_clips( - directory: str, json_data: Dict -) -> Tuple[float, List[Tuple[str, float]], List[Dict]]: - """Extract chunk URLs, start times, and presentation slides from the API JSON. - Returns: - (total_duration, [(url, start_time, conf_id), ...], [slide_event, ...]) +# --- Backward-compatible module-level functions --- - Each slide_event is: - {"time": float, "slide_number": int, "slide_url": str} - """ +def process_and_download_clips(directory, json_data): + """Parse API JSON for chunks, slides, admin IDs.""" + segments_builder = SegmentBuilder(FFmpegRunner.__new__(FFmpegRunner), MediaProber()) total_duration = float(json_data.get('duration', 0)) if not total_duration: raise ValueError('Duration not found in JSON data.') - admin_conf_ids = _extract_admin_conf_ids(json_data) + admin_conf_ids = segments_builder.extract_admin_conf_ids(json_data) - # Parse mediasession events for precise timing - mediasession_adds = {} # id -> {url, start, conf_id} + mediasession_adds = {} chunks = [] slide_events = [] for event in json_data.get('eventLogs', []): @@ -798,7 +538,6 @@ def process_and_download_clips( if not isinstance(data, dict): continue - # mediasession.add: precise file start time if module == 'mediasession.add' and 'url' in data: ms_id = data.get('id') stream = data.get('stream', {}) @@ -814,13 +553,11 @@ def process_and_download_clips( 'api_duration': 0, } - # mediasession.update: duration if module == 'mediasession.update': ms_id = data.get('id') if ms_id in mediasession_adds: mediasession_adds[ms_id]['api_duration'] = data.get('duration', 0) - # Fallback: any event with URL (catches non-mediasession chunks) elif 'url' in data and module not in ('mediasession.add',): stream = data.get('stream', {}) conf_id = None @@ -828,12 +565,10 @@ def process_and_download_clips( conf = stream.get('conference', {}) if isinstance(conf, dict): conf_id = conf.get('id') - # Only add if not already in mediasession_adds url = data['url'] if not any(ms['url'] == url for ms in mediasession_adds.values()): chunks.append((url, event.get('relativeTime', 0), conf_id, 0)) - # Presentation slide changes if module == 'presentation.update': fr = data.get('fileReference', {}) if not isinstance(fr, dict): @@ -841,14 +576,12 @@ def process_and_download_clips( slide = fr.get('slide', {}) if not isinstance(slide, dict) or not slide.get('url'): continue - slide_url = slide['url'] slide_events.append({ 'time': event.get('relativeTime', 0), 'slide_number': slide.get('number', 0), - 'slide_url': slide_url, + 'slide_url': slide['url'], }) - # Build tagged chunks from mediasession data (preferred) + fallback chunks tagged_chunks = [] for ms in mediasession_adds.values(): tagged_chunks.append(( @@ -859,7 +592,6 @@ def process_and_download_clips( for url, start, conf_id, api_dur in chunks: tagged_chunks.append((url, start, conf_id, conf_id in admin_conf_ids, api_dur)) - # Deduplicate consecutive identical slides deduped_slides = [] for se in slide_events: if not deduped_slides or se['slide_url'] != deduped_slides[-1]['slide_url']: @@ -871,1045 +603,17 @@ def process_and_download_clips( return total_duration, tagged_chunks, deduped_slides -def _deduplicate_overlapping( - video_files: list, -) -> List[Tuple[str, float]]: - """Remove overlapping video segments, keeping one per time window. - - Recordings often have parallel streams (multiple webcams) at the - same timestamp. Laying them out sequentially inflates the duration. - - Strategy: - 1. Pick the "main" conference: prefer ADMIN user conferences, - then fall back to the one with the most total recorded duration. - 2. Keep only segments from the main conference. - 3. For time gaps where the main conference has no video, fill in - with the longest available segment from any other conference. - """ - if not video_files: - return video_files - - # Annotate with duration, conf_id, and is_admin - from collections import defaultdict - annotated = [] # (path, start, duration, end, conf_id, is_admin) - for item in video_files: - path, start = item[0], item[1] - conf_id = item[2] if len(item) > 2 else None - is_admin = item[3] if len(item) > 3 else False - dur = _get_duration(path) - annotated.append((path, start, dur, start + dur, conf_id, is_admin)) - - # Score each conference by total duration - conf_total_dur = defaultdict(float) - conf_is_admin = {} - for _, _, dur, _, conf_id, is_admin in annotated: - if conf_id: - conf_total_dur[conf_id] += dur - if is_admin: - conf_is_admin[conf_id] = True - - # Pick main conference: ADMIN with most duration, else any with most duration - admin_confs = {c for c in conf_total_dur if conf_is_admin.get(c)} - if admin_confs: - main_conf = max(admin_confs, key=conf_total_dur.get) - logging.info( - f'Dedup: main conference {main_conf} (ADMIN, ' - f'{conf_total_dur[main_conf]:.0f}s total from ' - f'{sum(1 for x in annotated if x[4] == main_conf)} segments)' - ) - elif conf_total_dur: - main_conf = max(conf_total_dur, key=conf_total_dur.get) - logging.info( - f'Dedup: main conference {main_conf} ' - f'({conf_total_dur[main_conf]:.0f}s total from ' - f'{sum(1 for x in annotated if x[4] == main_conf)} segments)' - ) - else: - main_conf = None - - # Separate main conference segments from others - main_segs = sorted( - [s for s in annotated if s[4] == main_conf], - key=lambda x: x[1], - ) - other_segs = sorted( - [s for s in annotated if s[4] != main_conf], - key=lambda x: x[1], - ) - - # Build timeline from main conference segments (merge overlapping) - kept = [] - for seg in main_segs: - if not kept or seg[1] >= kept[-1][3] - 0.5: - kept.append(seg) - else: - # Overlapping main segments — keep the longer one - if seg[2] > kept[-1][2]: - kept[-1] = seg - - # Fill gaps with best available from other conferences - filled = [] - for i, seg in enumerate(kept): - gap_start = kept[i - 1][3] if i > 0 else 0 - gap_end = seg[1] - if gap_end - gap_start > 1.0: - # Find the longest other-conference segment covering this gap - best = None - for other in other_segs: - if other[3] <= gap_start or other[1] >= gap_end: - continue - # Compute how much of this segment actually covers the gap - overlap_start = max(other[1], gap_start) - overlap_end = min(other[3], gap_end) - overlap_dur = overlap_end - overlap_start - if overlap_dur <= 0: - continue - if best is None or overlap_dur > best[2]: - best = (other[0], overlap_start, overlap_dur, - overlap_end, other[4], other[5]) - if best: - filled.append(best) - filled.append(seg) - - original = len(video_files) - deduped = len(filled) - if original != deduped: - logging.info(f'Dedup: {original} -> {deduped} segments ' - f'(removed {original - deduped} overlapping)') - - return [(path, start) for path, start, dur, end, conf_id, is_admin in filled] - - -def _composite_slides( - video_path: str, - slide_events: List[Dict], - output_path: str, - tmp_dir: str, - total_duration: float, -) -> str: - """Composite presentation slides with webcam video. - - Layout (1280x720): - - Left 960px: presentation slide - - Right 320px, top: webcam (320x180) - - Right 320px, below webcam: black - """ - CANVAS_W, CANVAS_H = 1280, 720 - SLIDE_W, SLIDE_H = 960, CANVAS_H - CAM_W = CANVAS_W - SLIDE_W # 320 - - logging.info(f'Compositing {len(slide_events)} slide changes onto video...') - - # Step 1: Create a slide video track using concat demuxer. - # Each slide becomes a segment of the right duration. - slides_dir = os.path.join(tmp_dir, 'slide_segments') - os.makedirs(slides_dir, exist_ok=True) - - slide_segments = [] - for i, se in enumerate(slide_events): - t_start = se['time'] - t_end = slide_events[i + 1]['time'] if i + 1 < len(slide_events) else total_duration - duration = t_end - t_start - if duration <= 0: - continue - - seg_path = os.path.join(slides_dir, f'seg_{i}.mp4') - # Use -frames:v to strictly limit frame count at 1fps. - # This avoids runaway encoding for long durations. - n_frames = max(1, int(duration)) - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-loop', '1', '-framerate', '1', - '-i', se['local_path'], - '-vf', f'scale={SLIDE_W}:{SLIDE_H}:force_original_aspect_ratio=decrease,' - f'pad={SLIDE_W}:{SLIDE_H}:(ow-iw)/2:(oh-ih)/2:white', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - '-r', '1', '-frames:v', str(n_frames), - seg_path, - ], - description=f'slide segment {i+1}/{len(slide_events)} ' - f'({n_frames} frames, {duration:.0f}s)', - ) - slide_segments.append(seg_path) - - # Add black leader if first slide starts after 0 - first_time = slide_events[0]['time'] if slide_events else 0 - if first_time > 0.5: - leader_path = os.path.join(slides_dir, 'leader.mp4') - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-f', 'lavfi', '-i', - f'color=c=black:s={SLIDE_W}x{SLIDE_H}:d={first_time}:r=1', - *_get_video_encoder_fast(), - '-pix_fmt', 'yuv420p', - leader_path, - ], - description='slide leader (black)', - ) - slide_segments.insert(0, leader_path) - - # Concatenate slide segments into one track - slide_track_path = os.path.join(tmp_dir, 'slide_track.mp4') - concat_list = os.path.join(slides_dir, 'concat.txt') - with open(concat_list, 'w') as f: - for seg in slide_segments: - f.write(f"file '{os.path.abspath(seg)}'\n") - - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-f', 'concat', '-safe', '0', '-i', concat_list, - '-c', 'copy', slide_track_path, - ], - description='concat slide track', - ) - logging.info('Slide track created') - - # Step 2: Combine slide track + webcam into final layout. - # Process in chunks to avoid OOM on long videos. - CHUNK_SECS = 1800 # 30 minutes per chunk - - _detect_gpu() - if _CUDA_OVERLAY_AVAILABLE: - # Build canvas on CPU (slide left, webcam top-right on black bg) - # then upload to CUDA for encoding. overlay_cuda has limited layout - # control, so we use CPU overlay for correct positioning. - filter_graph = ( - f'[0:v]fps=25,scale={CAM_W}:-2,setsar=1[webcam];' - f'[1:v]fps=25,scale={SLIDE_W}:{CANVAS_H}:force_original_aspect_ratio=decrease,' - f'pad={SLIDE_W}:{CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' - f'color=c=black:s={CANVAS_W}x{CANVAS_H}:r=25[bg];' - f'[bg][slide]overlay=0:0[tmp];' - f'[tmp][webcam]overlay={SLIDE_W}:0[out]' - ) - else: - filter_graph = ( - f'[0:v]fps=25,scale={CAM_W}:-2,setsar=1[webcam];' - f'[1:v]fps=25,scale={SLIDE_W}:{CANVAS_H}:force_original_aspect_ratio=decrease,' - f'pad={SLIDE_W}:{CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' - f'color=c=black:s={CANVAS_W}x{CANVAS_H}:r=25[bg];' - f'[bg][slide]overlay=0:0[tmp];' - f'[tmp][webcam]overlay={SLIDE_W}:0[out]' - ) - - n_chunks = max(1, int(total_duration + CHUNK_SECS - 1) // CHUNK_SECS) - - enc_args = _get_video_encoder() - - if n_chunks == 1: - # Short video — composite in one pass - cmd = [ - 'ffmpeg', '-y', '-v', 'warning', - '-i', video_path, - '-i', slide_track_path, - '-filter_complex', filter_graph, - '-map', '[out]', '-map', '0:a?', - *enc_args, - '-c:a', 'copy', - '-r', '25', - '-shortest', - output_path, - ] - _run_ffmpeg(cmd, description='composite slides + webcam') - else: - # Long video — split into chunks, composite each, then concat - logging.info(f'Compositing in {n_chunks} chunks of {CHUNK_SECS}s to avoid OOM') - chunks_dir = os.path.join(tmp_dir, 'composite_chunks') - os.makedirs(chunks_dir, exist_ok=True) - chunk_paths = [] - - for ci in range(n_chunks): - ss = ci * CHUNK_SECS - chunk_path = os.path.join(chunks_dir, f'chunk_{ci:03d}.mp4') - cmd = [ - 'ffmpeg', '-y', '-v', 'warning', - '-ss', str(ss), '-t', str(CHUNK_SECS), - '-i', video_path, - '-ss', str(ss), '-t', str(CHUNK_SECS), - '-i', slide_track_path, - '-filter_complex', filter_graph, - '-map', '[out]', '-map', '0:a?', - *enc_args, - '-c:a', 'aac', '-b:a', '192k', - '-r', '25', - '-shortest', - chunk_path, - ] - _run_ffmpeg(cmd, description=f'composite chunk {ci+1}/{n_chunks}') - chunk_paths.append(chunk_path) - logging.info(f'Composite chunk {ci+1}/{n_chunks} done') - - # Concatenate chunks - chunk_list = os.path.join(chunks_dir, 'concat.txt') - with open(chunk_list, 'w') as f: - for cp in chunk_paths: - f.write(f"file '{os.path.abspath(cp)}'\n") - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - '-f', 'concat', '-safe', '0', '-i', chunk_list, - '-c', 'copy', output_path, - ], - description='concat composite chunks', - ) - shutil.rmtree(chunks_dir, ignore_errors=True) - - logging.info('Slide compositing complete') - - # Cleanup slide segments - shutil.rmtree(slides_dir, ignore_errors=True) - return output_path - - -def _probe_file(file_path: str) -> dict: - """Probe a single file and return all metadata needed for planning.""" - info = _ffprobe_streams(file_path) - streams = info.get('streams', []) - fmt = info.get('format', {}) - - has_video = any(s.get('codec_type') == 'video' for s in streams) - has_audio = any(s.get('codec_type') == 'audio' for s in streams) - - width, height, pix_fmt = 0, 0, 'yuv420p' - for s in streams: - if s.get('codec_type') == 'video': - width = int(s.get('width', 0)) - height = int(s.get('height', 0)) - pix_fmt = s.get('pix_fmt', 'yuv420p') - break - - duration = 0.0 - if 'duration' in fmt: - duration = float(fmt['duration']) - else: - for s in streams: - if 'duration' in s: - duration = float(s['duration']) - break - - valid = bool(streams) and duration > 0 - - return { - 'path': file_path, - 'valid': valid, - 'has_video': has_video, - 'has_audio': has_audio, - 'width': width, - 'height': height, - 'pix_fmt': pix_fmt, - 'duration': duration, - } - - -def _probe_all_files(downloaded_files: list) -> List[dict]: - """Probe all files in parallel. Returns list of file info dicts.""" - results = [] - paths = [(item[0], item[1], - item[2] if len(item) > 2 else None, - item[3] if len(item) > 3 else False, - item[4] if len(item) > 4 else 0) - for item in downloaded_files] - - with ThreadPoolExecutor(max_workers=8) as pool: - futures = {} - for path, start_time, conf_id, is_admin, api_duration in paths: - fut = pool.submit(_probe_file, path) - futures[fut] = (start_time, conf_id, is_admin, api_duration) - for fut in as_completed(futures): - start_time, conf_id, is_admin, api_duration = futures[fut] - info = fut.result() - info['start_time'] = start_time - info['conf_id'] = conf_id - info['is_admin'] = is_admin - # Use API duration if available (more accurate than ffprobe) - if api_duration > 0: - info['api_duration'] = api_duration - results.append(info) - - results.sort(key=lambda x: x['start_time']) - return results - - -def analyze_video( - total_duration: float, - downloaded_files: list, - directory: str, - output_path: str, - max_duration=None, - slide_events=None, -) -> dict: - """Phase 1: Probe all files, make all decisions, write manifest. - - Fast — only ffprobe calls, no re-encoding. Returns manifest dict - and writes it to {directory}/manifest.json. - """ - _check_ffmpeg() - logging.info('Phase 1: Analyzing files...') - - # Probe all files in parallel - all_files = _probe_all_files(downloaded_files) - - # Classify - video_files = [] # dicts with has_video=True - audio_files = [] # dicts with has_video=False (audio-only) - skipped = 0 - errors = [] - - for f in all_files: - if not f['valid']: - skipped += 1 - errors.append(f'Corrupt/invalid: {f["path"]}') - continue - if f['has_video']: - video_files.append(f) - else: - audio_files.append(f) - - if skipped: - logging.warning(f'Skipped {skipped} corrupt/invalid files') - logging.info(f'Segments: {len(video_files)} video, {len(audio_files)} audio-only') - - if not video_files: - raise ValueError('No valid video segments found.') - - # Determine target resolution - target_w, target_h, target_pix_fmt = 0, 0, 'yuv420p' - for f in video_files: - if f['width'] * f['height'] > target_w * target_h: - target_w, target_h, target_pix_fmt = f['width'], f['height'], f['pix_fmt'] - if target_w * target_h < 640 * 360: - target_w, target_h = 640, 360 - MAX_H = 720 - if target_h > MAX_H: - target_w = int(target_w * MAX_H / target_h) - target_w -= target_w % 2 - target_h = MAX_H - logging.info(f'Target resolution: {target_w}x{target_h}, pix_fmt={target_pix_fmt}') - - # Detect overlaps using cached durations - has_overlaps = False - for i in range(1, len(video_files)): - prev_dur = video_files[i-1].get('api_duration', 0) or video_files[i-1]['duration'] - prev_end = video_files[i-1]['start_time'] + prev_dur - if video_files[i]['start_time'] < prev_end - 0.5: - has_overlaps = True - break - - # Decide overlap strategy and build segment plan - overlap_strategy = 'none' - kept_video = [] # files to use as video segments - extra_audio = [] # webcam files dropped but with audio to extract - segments = [] # populated by grid, or built later for dedup/none - - if has_overlaps and slide_events: - overlap_strategy = 'dedup' - # Use _deduplicate_overlapping with our probed data - vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), - f.get('is_admin', False)) for f in video_files] - deduped = _deduplicate_overlapping(vf_tuples) - kept_paths = {v[0] for v in deduped} - - # Map back to file info dicts - dedup_map = {f['path']: f for f in video_files} - for path, start_time in deduped: - if path in dedup_map: - kept_video.append(dedup_map[path]) - - # Find dropped webcams with audio - for f in video_files: - if f['path'] not in kept_paths and f['has_audio']: - extra_audio.append(f) - - logging.info(f'Dedup: kept {len(kept_video)}, ' - f'{len(extra_audio)} extra audio from webcams') - elif has_overlaps: - overlap_strategy = 'grid' - # Plan grid windows using probed durations - # Build annotated list: (path, start, duration, end) - annotated = [] - for f in video_files: - dur = f.get('api_duration', 0) or f['duration'] - annotated.append((f['path'], f['start_time'], dur, - f['start_time'] + dur, f['has_audio'])) - # Collect all event times - event_times_set = set() - for _, start, _, end, _ in annotated: - event_times_set.add(start) - event_times_set.add(end) - event_times_set.add(total_duration) - event_times = sorted(event_times_set) - - # Build grid windows with validated sources - grid_windows = [] - prev_active_key = None - window_start = None - window_active = None - for ei in range(len(event_times) - 1): - t_start = event_times[ei] - t_end = event_times[ei + 1] - if t_end - t_start < 0.1: - continue - active = [] - for path, seg_start, dur, seg_end, has_audio in annotated: - if seg_start < t_end and seg_end > t_start: - offset = max(0, t_start - seg_start) - remaining = dur - offset - if remaining > 0.5: - active.append({'path': path, 'offset': offset, - 'remaining': remaining, - 'has_audio': has_audio}) - active_key = tuple(a['path'] for a in active) - if active_key == prev_active_key and window_start is not None: - continue - if prev_active_key is not None and window_start is not None: - grid_windows.append({ - 'start_time': window_start, - 'duration': t_start - window_start, - 'sources': window_active, - }) - window_start = t_start - prev_active_key = active_key - window_active = active - if window_start is not None and window_active: - grid_windows.append({ - 'start_time': window_start, - 'duration': event_times[-1] - window_start, - 'sources': window_active, - }) - - # Convert grid windows to segments - for gw in grid_windows: - if not gw['sources']: - segments.append({ - 'type': 'gap', - 'duration': gw['duration'], - 'start_time': gw['start_time'], - }) - else: - segments.append({ - 'type': 'grid', - 'start_time': gw['start_time'], - 'planned_duration': gw['duration'], - 'sources': gw['sources'], - }) - # Extract audio from ALL video files with audio for mixing - # Grid only keeps audio from input 0 — other webcam audio is lost - for f in video_files: - if f['has_audio']: - extra_audio.append(f) - if extra_audio: - logging.info(f'Grid: {len(extra_audio)} webcam audio tracks to extract') - # kept_video not used for grid — segments has everything - else: - vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), - f.get('is_admin', False)) for f in video_files] - deduped = _deduplicate_overlapping(vf_tuples) - dedup_map = {f['path']: f for f in video_files} - for path, start_time in deduped: - if path in dedup_map: - kept_video.append(dedup_map[path]) - - # Build segment plan (gaps + video segments in order) - # Grid strategy already has segments populated - if overlap_strategy != 'grid': - segments = [] - current_time = 0.0 - for i, f in enumerate(kept_video): - start_time = f['start_time'] - - if start_time < current_time - 0.5: - errors.append(f'Segment {i} overlaps at {start_time:.1f}s (current={current_time:.1f}s)') - continue - - gap = start_time - current_time - if gap > 0.1: - segments.append({ - 'type': 'gap', - 'duration': gap, - 'start_time': current_time, - }) - - # Compute max duration (truncate at next segment) - max_dur = 0 - for j in range(i + 1, len(kept_video)): - ns = kept_video[j]['start_time'] - if ns > start_time + 0.5: - max_dur = ns - start_time - break - - # planned_duration = how long this segment should be in the output - file_dur = f.get('api_duration', 0) or f['duration'] - planned_dur = max_dur if max_dur > 0 else file_dur - segments.append({ - 'type': 'video', - 'source_path': f['path'], - 'start_time': start_time, - 'source_duration': f['duration'], - 'max_duration': max_dur, - 'planned_duration': planned_dur, - 'has_audio': f['has_audio'], - 'width': f['width'], - 'height': f['height'], - }) - - current_time = start_time + planned_dur - - # For grid: fill timeline gaps so video matches audio timeline exactly - if overlap_strategy == 'grid' and segments: - filled = [] - current_time = 0.0 - for s in segments: - start = s.get('start_time', 0) - if start - current_time > 0.1: - filled.append({ - 'type': 'gap', - 'duration': start - current_time, - 'start_time': current_time, - }) - filled.append(s) - current_time = start + s.get('planned_duration', s.get('duration', 0)) - segments = filled - - # Trailing gap — extend video to total_duration so audio can play over black - if overlap_strategy == 'grid': - current_time = 0 - for s in segments: - end = s.get('start_time', 0) + s.get('planned_duration', s.get('duration', 0)) - if end > current_time: - current_time = end - if current_time < total_duration - 0.1: - segments.append({ - 'type': 'gap', - 'duration': total_duration - current_time, - 'start_time': current_time, - }) - - # Audio merge plan - all_audio = [{'path': f['path'], 'start_time': f['start_time'], - 'origin': 'original'} for f in audio_files] - for f in extra_audio: - all_audio.append({ - 'path': f['path'], - 'start_time': f['start_time'], - 'origin': 'webcam_extract', - }) - - # Slide compositing plan - slide_plan = None - if slide_events: - CHUNK_SECS = 1800 - n_chunks = max(1, int(total_duration + CHUNK_SECS - 1) // CHUNK_SECS) - slide_plan = { - 'events': slide_events, - 'chunk_seconds': CHUNK_SECS, - 'num_chunks': n_chunks, - } +def compile_final_video(total_duration, downloaded_files, directory, + output_path, max_duration, slide_events=None): + """Backward-compatible entry point.""" + processor = VideoProcessor() + processor.process(total_duration, downloaded_files, directory, + output_path, max_duration, slide_events) - # GPU detection - _detect_gpu() - gpu = { - 'nvenc': bool(_NVENC_AVAILABLE), - 'cuda_overlay': bool(_CUDA_OVERLAY_AVAILABLE), - } - - manifest = { - 'version': 1, - 'total_duration': total_duration, - 'directory': directory, - 'output_path': output_path, - 'max_duration': max_duration, - 'target': { - 'width': target_w, - 'height': target_h, - 'pix_fmt': target_pix_fmt, - }, - 'overlap_strategy': overlap_strategy, - 'segments': segments, - 'audio_tracks': all_audio, - 'slide_compositing': slide_plan, - 'gpu': gpu, - 'errors': errors, - 'warnings': [], - 'stats': { - 'total_files': len(all_files), - 'video_files': len(video_files), - 'audio_files': len(audio_files), - 'kept_video': len(kept_video), - 'extra_audio': len(extra_audio), - 'skipped': skipped, - 'segments': len(segments), - 'gaps': sum(1 for s in segments if s['type'] == 'gap'), - }, - } - - # Write manifest - manifest_path = os.path.join(directory, 'manifest.json') - with open(manifest_path, 'w') as f: - json.dump(manifest, f, indent=2, ensure_ascii=False) - logging.info(f'Manifest written to {manifest_path}') - logging.info(f'Plan: {manifest["stats"]["segments"]} segments ' - f'({manifest["stats"]["gaps"]} gaps), ' - f'{len(all_audio)} audio tracks, ' - f'strategy={overlap_strategy}') - - # Validate: check for segments where source is shorter than planned - warnings = [] - for seg in segments: - if seg['type'] != 'video': - continue - src_dur = seg['source_duration'] - planned = seg['planned_duration'] - if src_dur < planned - 1.0: - warnings.append( - f'Segment at {seg["start_time"]:.0f}s: source={src_dur:.0f}s ' - f'but planned={planned:.0f}s (will pad {planned-src_dur:.0f}s black)' - ) - manifest['warnings'] = warnings - - if errors: - for e in errors: - logging.warning(f'Analyze: {e}') - if warnings: - for w in warnings: - logging.warning(f'Analyze: {w}') - - return manifest - - -def execute_video(manifest: dict): - """Phase 2: Execute the plan from the manifest. No probing, no decisions.""" - logging.info('Phase 2: Executing plan...') - - directory = manifest['directory'] - output_path = manifest['output_path'] - total_duration = manifest['total_duration'] - target = manifest['target'] - target_w, target_h = target['width'], target['height'] - target_pix_fmt = target['pix_fmt'] - - tmp_dir = os.path.join(directory, '_tmp_ffmpeg') - os.makedirs(tmp_dir, exist_ok=True) - - # Step 1: Extract audio from dropped webcam files - audio_files = [] # (path, start_time) for merge step - extra_dir = os.path.join(tmp_dir, 'extra_audio') - os.makedirs(extra_dir, exist_ok=True) - extract_count = 0 - for track in manifest['audio_tracks']: - if track['origin'] == 'webcam_extract': - audio_path = os.path.join(extra_dir, f'extra_{extract_count}.m4a') - try: - _run_ffmpeg( - ['ffmpeg', '-y', '-v', 'error', - '-i', track['path'], '-vn', - '-c:a', 'aac', '-b:a', '128k', - '-ar', '44100', '-ac', '2', - audio_path], - description=f'extract audio from webcam {extract_count}', - ) - audio_files.append((audio_path, track['start_time'])) - extract_count += 1 - except subprocess.CalledProcessError: - logging.warning(f'Failed to extract audio from {track["path"]}') - else: - audio_files.append((track['path'], track['start_time'])) - if extract_count: - logging.info(f'Extracted audio from {extract_count} webcam files') - - # Step 2: Process segments - segments = manifest['segments'] - grid_dir = os.path.join(tmp_dir, 'grid_segments') - concat_segments = [] - for i, seg in enumerate(segments): - if seg['type'] == 'gap': - gap_path = os.path.join(tmp_dir, f'gap_{i}.mp4') - _generate_black_segment(gap_path, seg['duration'], - target_w, target_h, target_pix_fmt) - concat_segments.append(gap_path) - logging.info(f'Generated {seg["duration"]:.1f}s black gap') - elif seg['type'] == 'grid': - os.makedirs(grid_dir, exist_ok=True) - seg_path = os.path.join(grid_dir, f'grid_{i}.mp4') - duration = seg['planned_duration'] - # Sort sources: audio-bearing first so input 0 has audio - sources = sorted(seg['sources'], - key=lambda s: not s.get('has_audio', False)) - active = [(s['path'], s['offset']) for s in sources] - try: - if len(active) == 1: - path, offset = active[0] - _run_ffmpeg( - ['ffmpeg', '-y', '-v', 'error', - '-ss', str(offset), '-i', path, - '-t', str(duration), - '-vf', f'scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' - f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', - *_get_video_encoder_fast(), '-pix_fmt', 'yuv420p', - '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', - '-r', '25', seg_path], - description=f'grid {i} (1 webcam, {duration:.0f}s)', - ) - else: - _composite_grid(active, duration, seg_path, target_w, target_h) - with_audio = os.path.join(tmp_dir, f'grida_{i}.mp4') - final_seg = _ensure_audio_stream(seg_path, with_audio) - concat_segments.append(final_seg) - except subprocess.CalledProcessError: - logging.warning(f'Grid segment {i} failed, using black gap') - gap_path = os.path.join(tmp_dir, f'gridfail_{i}.mp4') - _generate_black_segment(gap_path, duration, - target_w, target_h, target_pix_fmt) - concat_segments.append(gap_path) - elif seg['type'] == 'video': - norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') - _normalize_segment(seg['source_path'], norm_path, - target_w, target_h, target_pix_fmt, - max_duration=seg.get('max_duration', 0)) - with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') - final_seg = _ensure_audio_stream(norm_path, with_audio_path) - - # Verify duration matches plan — pad with black if too short - actual_dur = _get_duration(final_seg) - planned_dur = seg.get('planned_duration', 0) - if planned_dur > 0 and actual_dur < planned_dur - 1.0: - shortfall = planned_dur - actual_dur - logging.warning( - f'Segment {i} is {actual_dur:.1f}s but planned ' - f'{planned_dur:.1f}s — padding {shortfall:.1f}s' - ) - pad_path = os.path.join(tmp_dir, f'pad_{i}.mp4') - _generate_black_segment(pad_path, shortfall, - target_w, target_h, target_pix_fmt) - concat_segments.append(final_seg) - concat_segments.append(pad_path) - else: - concat_segments.append(final_seg) - - # Step 4: Concatenate - concat_list_path = os.path.join(tmp_dir, 'concat.txt') - with open(concat_list_path, 'w') as f: - for seg in concat_segments: - f.write(f"file '{os.path.abspath(seg)}'\n") - - video_only_path = os.path.join(tmp_dir, 'video_concat.mp4') - logging.info(f'Concatenating {len(concat_segments)} segments...') - - cap = manifest.get('max_duration') or total_duration - concat_cmd = [ - 'ffmpeg', '-y', '-v', 'warning', - '-f', 'concat', '-safe', '0', - '-i', concat_list_path, - '-c', 'copy', - ] - if cap: - concat_cmd.extend(['-t', str(cap)]) - concat_cmd.append(video_only_path) - _run_ffmpeg(concat_cmd, description=f'concat {len(concat_segments)} segments') - - # Step 5: Composite slides - slide_plan = manifest.get('slide_compositing') - if slide_plan and slide_plan.get('events'): - composited_path = os.path.join(tmp_dir, 'video_composited.mp4') - _composite_slides( - video_only_path, slide_plan['events'], composited_path, - tmp_dir, total_duration, - ) - video_only_path = composited_path - - # Step 6: Merge audio - if audio_files: - logging.info(f'Merging {len(audio_files)} audio tracks...') - result_path = _merge_audio_tracks( - video_only_path, audio_files, tmp_dir, output_path, - total_duration, - ) - else: - shutil.move(video_only_path, output_path) - result_path = output_path - - # Cleanup - shutil.rmtree(tmp_dir, ignore_errors=True) - logging.info(f'Final video saved to {result_path}') - return result_path - - -def compile_final_video( - total_duration: float, - downloaded_files: List[Tuple[str, float]], - directory: str, - output_path: str, - max_duration: Union[int, None], - slide_events: List[Dict] = None, -): - """Two-phase video compilation: analyze then execute.""" - manifest = analyze_video( - total_duration, downloaded_files, directory, output_path, - max_duration, slide_events, - ) - execute_video(manifest) - - -def _merge_audio_tracks( - video_path: str, - audio_files: List[Tuple[str, float]], - tmp_dir: str, - output_path: str, - total_duration: float = 0, -) -> str: - """Merge audio-only tracks on top of the concatenated video. - - To avoid OOM (too many ffmpeg inputs) and disk exhaustion (huge WAVs), - we mix audio in small batches directly with adelay inside the filter, - outputting compressed m4a. Each batch produces one small file, and - consumed intermediates are deleted immediately. - - Strategy: - 1. Mix audio files in batches of AUDIO_MERGE_BATCH_SIZE, applying - adelay inside the filter graph (no pre-materialized delayed files). - 2. Tree-reduce batch outputs until one mixed track remains. - 3. Overlay the single mixed audio onto the video. - """ - video_duration = total_duration or _get_duration(video_path) - batch_size = AUDIO_MERGE_BATCH_SIZE - - # Step 1: Mix in batches with inline adelay (no intermediate WAVs). - # Each batch takes up to batch_size original audio files, applies - # adelay per track, mixes them, and writes a compressed m4a. - batch_outputs = [] # (path, effective_start=0 since delay is baked in) - for batch_idx, batch_start in enumerate( - range(0, len(audio_files), batch_size) - ): - batch = audio_files[batch_start:batch_start + batch_size] - batch_out = os.path.join(tmp_dir, f'audio_batch_{batch_idx}.m4a') - - inputs = [] - filter_parts = [] - mix_labels = [] - for j, (apath, start_time) in enumerate(batch): - inputs.extend(['-i', apath]) - delay_ms = int(start_time * 1000) - label = f'a{j}' - filter_parts.append( - f'[{j}:a]adelay={delay_ms}|{delay_ms},' - f'apad=whole_dur={video_duration},' - f'atrim=0:{video_duration},' - f'asetpts=PTS-STARTPTS[{label}]' - ) - mix_labels.append(f'[{label}]') - if len(batch) == 1: - # Single track — just delay and encode, no amix needed - filter_graph = filter_parts[0].rsplit('[', 1)[0] # strip label - else: - # normalize=0 prevents amix from dividing by N - filter_graph = ( - ';'.join(filter_parts) + ';' - + ''.join(mix_labels) - + f'amix=inputs={len(batch)}:duration=longest:normalize=0' - ) - - try: - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - *inputs, - '-filter_complex', filter_graph, - '-c:a', 'aac', '-b:a', '128k', - '-ar', '44100', '-ac', '2', - batch_out, - ], - description=f'amix batch {batch_idx+1} ' - f'({len(batch)} tracks, offset {batch[0][1]:.0f}-' - f'{batch[-1][1]:.0f}s)', - ) - batch_outputs.append(batch_out) - except subprocess.CalledProcessError: - logging.warning( - f'Audio batch {batch_idx+1} failed, skipping ' - f'{len(batch)} tracks' - ) - logging.info( - f'Audio batch {batch_idx+1}/' - f'{(len(audio_files) + batch_size - 1) // batch_size} done' - ) - - # If no batches succeeded, skip audio overlay entirely. - if not batch_outputs: - logging.warning('All audio batches failed, skipping audio overlay') - shutil.move(video_path, output_path) - return output_path - - # Step 2: Tree-reduce batch outputs until one remains. - round_num = 0 - current_paths = batch_outputs - while len(current_paths) > 1: - round_num += 1 - next_paths = [] - for batch_start in range(0, len(current_paths), batch_size): - batch = current_paths[batch_start:batch_start + batch_size] - if len(batch) == 1: - next_paths.append(batch[0]) - continue +# Keep old names for backward compatibility with webinar.py +def analyze_video(*args, **kwargs): + return VideoProcessor().analyze(*args, **kwargs) - batch_out = os.path.join( - tmp_dir, f'audio_reduce_r{round_num}_b{batch_start}.m4a' - ) - inputs = [] - for bp in batch: - inputs.extend(['-i', bp]) - - labels = ''.join(f'[{j}:a]' for j in range(len(batch))) - amix_filter = ( - f'{labels}amix=inputs={len(batch)}' - f':duration=longest:normalize=0' - ) - - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'error', - *inputs, - '-filter_complex', amix_filter, - '-c:a', 'aac', '-b:a', '128k', - '-ar', '44100', '-ac', '2', - batch_out, - ], - description=f'amix reduce round {round_num}, ' - f'{len(batch)} tracks', - ) - next_paths.append(batch_out) - - # Delete consumed intermediates - for bp in batch: - try: - os.remove(bp) - except OSError: - pass - - logging.info( - f'Audio reduce round {round_num}: {len(current_paths)} -> ' - f'{len(next_paths)} tracks' - ) - current_paths = next_paths - - mixed_audio_path = current_paths[0] - - # Step 3: Overlay the single mixed audio track onto the video. - logging.info('Overlaying mixed audio onto video...') - _run_ffmpeg( - [ - 'ffmpeg', '-y', '-v', 'warning', - '-i', video_path, - '-i', mixed_audio_path, - '-filter_complex', - '[0:a][1:a]amix=inputs=2:duration=first:normalize=0[aout]', - '-map', '0:v', '-map', '[aout]', - '-c:v', 'copy', - '-c:a', 'aac', '-b:a', '192k', - output_path, - ], - description='overlay mixed audio onto video', - ) - return output_path +def execute_video(manifest): + return VideoProcessor().execute(manifest) diff --git a/mtslinker/segments.py b/mtslinker/segments.py new file mode 100644 index 0000000..911884f --- /dev/null +++ b/mtslinker/segments.py @@ -0,0 +1,192 @@ +import logging +import os +from collections import defaultdict +from typing import Dict, List, Tuple + +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber + + +class SegmentBuilder: + """Builds, normalizes, and deduplicates video segments.""" + + def __init__(self, ffmpeg: FFmpegRunner, prober: MediaProber): + self.ffmpeg = ffmpeg + self.prober = prober + + def generate_black(self, output_path: str, duration: float, + width: int = 1920, height: int = 1080, + pix_fmt: str = 'yuv420p') -> str: + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', f'color=c=black:s={width}x{height}:d={duration}:r=25', + '-f', 'lavfi', '-i', f'anullsrc=r=44100:cl=stereo', + '-t', str(duration), + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', pix_fmt, + '-c:a', 'aac', '-b:a', '128k', + '-shortest', + output_path, + ], + description=f'generate black segment ({duration:.1f}s)', + ) + return output_path + + def ensure_audio(self, input_path: str, output_path: str) -> str: + info = self.prober.probe_streams(input_path) + has_audio = any( + s.get('codec_type') == 'audio' for s in info.get('streams', []) + ) + if has_audio: + return input_path + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-i', input_path, + '-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo', + '-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k', + '-shortest', + output_path, + ], + description='add silent audio stream', + ) + return output_path + + def normalize(self, input_path: str, output_path: str, + width: int, height: int, pix_fmt: str, + max_duration: float = 0) -> str: + duration_args = ['-t', str(max_duration)] if max_duration > 0 else [] + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-i', input_path, + '-vf', f'scale={width}:{height}:force_original_aspect_ratio=decrease,' + f'pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + '-pix_fmt', pix_fmt, + *self.ffmpeg.get_video_encoder_fast(), + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + *duration_args, + output_path, + ], + description=f'normalize segment {os.path.basename(input_path)}', + ) + return output_path + + def deduplicate(self, video_files: list) -> List[Tuple[str, float]]: + if not video_files: + return video_files + + annotated = [] + for item in video_files: + path, start = item[0], item[1] + conf_id = item[2] if len(item) > 2 else None + is_admin = item[3] if len(item) > 3 else False + dur = self.prober.get_duration(path) + annotated.append((path, start, dur, start + dur, conf_id, is_admin)) + + conf_total_dur = defaultdict(float) + conf_is_admin = {} + for _, _, dur, _, conf_id, is_admin in annotated: + if conf_id: + conf_total_dur[conf_id] += dur + if is_admin: + conf_is_admin[conf_id] = True + + admin_confs = {c for c in conf_total_dur if conf_is_admin.get(c)} + if admin_confs: + main_conf = max(admin_confs, key=conf_total_dur.get) + logging.info( + f'Dedup: main conference {main_conf} (ADMIN, ' + f'{conf_total_dur[main_conf]:.0f}s total from ' + f'{sum(1 for x in annotated if x[4] == main_conf)} segments)' + ) + elif conf_total_dur: + main_conf = max(conf_total_dur, key=conf_total_dur.get) + logging.info( + f'Dedup: main conference {main_conf} ' + f'({conf_total_dur[main_conf]:.0f}s total from ' + f'{sum(1 for x in annotated if x[4] == main_conf)} segments)' + ) + else: + main_conf = None + + main_segs = sorted( + [s for s in annotated if s[4] == main_conf], key=lambda x: x[1]) + other_segs = sorted( + [s for s in annotated if s[4] != main_conf], key=lambda x: x[1]) + + kept = [] + for seg in main_segs: + if not kept or seg[1] >= kept[-1][3] - 0.5: + kept.append(seg) + else: + if seg[2] > kept[-1][2]: + kept[-1] = seg + + filled = [] + for i, seg in enumerate(kept): + gap_start = kept[i - 1][3] if i > 0 else 0 + gap_end = seg[1] + if gap_end - gap_start > 1.0: + best = None + for other in other_segs: + if other[3] <= gap_start or other[1] >= gap_end: + continue + overlap_start = max(other[1], gap_start) + overlap_end = min(other[3], gap_end) + overlap_dur = overlap_end - overlap_start + if overlap_dur <= 0: + continue + if best is None or overlap_dur > best[2]: + best = (other[0], overlap_start, overlap_dur, + overlap_end, other[4], other[5]) + if best: + filled.append(best) + filled.append(seg) + + result = [(path, start) for path, start, dur, end, conf_id, is_admin in filled] + logging.info(f'Dedup: {len(annotated)} -> {len(result)} segments ' + f'(removed {len(annotated) - len(result)} overlapping)') + return result + + def extract_admin_conf_ids(self, json_data: Dict) -> set: + admin_user_ids = set() + user_to_conf = {} + + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data_list = event.get('data', []) + if isinstance(data_list, dict): + data_list = [data_list] + if not isinstance(data_list, list): + continue + + for d in data_list: + if not isinstance(d, dict): + continue + if 'userlist' in module: + role = d.get('role', '') + user = d.get('user', {}) + if isinstance(user, dict) and role == 'ADMIN': + uid = user.get('id') + if uid: + admin_user_ids.add(uid) + if module == 'conference.add': + user = d.get('user', {}) + if isinstance(user, dict): + uid = user.get('id') + cid = d.get('id') + if uid and cid: + user_to_conf.setdefault(uid, set()).add(cid) + + admin_confs = set() + for uid in admin_user_ids: + admin_confs.update(user_to_conf.get(uid, set())) + + if admin_confs: + logging.info(f'Found {len(admin_confs)} conference(s) from ADMIN users') + return admin_confs diff --git a/mtslinker/slides.py b/mtslinker/slides.py new file mode 100644 index 0000000..772bd22 --- /dev/null +++ b/mtslinker/slides.py @@ -0,0 +1,153 @@ +import logging +import os +import shutil +from typing import Dict, List + +from mtslinker.ffmpeg import FFmpegRunner + + +class SlideCompositor: + """Composites presentation slides alongside webcam video.""" + + CANVAS_W = 1280 + CANVAS_H = 720 + SLIDE_W = 960 + CAM_W = 320 + CHUNK_SECS = 1800 + + def __init__(self, ffmpeg: FFmpegRunner): + self.ffmpeg = ffmpeg + + def composite(self, video_path: str, slide_events: List[Dict], + output_path: str, tmp_dir: str, total_duration: float) -> str: + logging.info(f'Compositing {len(slide_events)} slide changes onto video...') + + slide_track_path = self._build_slide_track(slide_events, tmp_dir, total_duration) + + filter_graph = ( + f'[0:v]fps=25,scale={self.CAM_W}:-2,setsar=1[webcam];' + f'[1:v]fps=25,scale={self.SLIDE_W}:{self.CANVAS_H}:' + f'force_original_aspect_ratio=decrease,' + f'pad={self.SLIDE_W}:{self.CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' + f'color=c=black:s={self.CANVAS_W}x{self.CANVAS_H}:r=25[bg];' + f'[bg][slide]overlay=0:0[tmp];' + f'[tmp][webcam]overlay={self.SLIDE_W}:0[out]' + ) + + n_chunks = max(1, int(total_duration + self.CHUNK_SECS - 1) // self.CHUNK_SECS) + enc_args = self.ffmpeg.get_video_encoder() + + if n_chunks == 1: + cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', video_path, '-i', slide_track_path, + '-filter_complex', filter_graph, + '-map', '[out]', '-map', '0:a?', + *enc_args, '-c:a', 'copy', '-r', '25', '-shortest', + output_path, + ] + self.ffmpeg.run(cmd, description='composite slides + webcam') + else: + logging.info(f'Compositing in {n_chunks} chunks of {self.CHUNK_SECS}s to avoid OOM') + chunks_dir = os.path.join(tmp_dir, 'composite_chunks') + os.makedirs(chunks_dir, exist_ok=True) + chunk_paths = [] + + for ci in range(n_chunks): + ss = ci * self.CHUNK_SECS + chunk_path = os.path.join(chunks_dir, f'chunk_{ci:03d}.mp4') + cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-ss', str(ss), '-t', str(self.CHUNK_SECS), + '-i', video_path, + '-ss', str(ss), '-t', str(self.CHUNK_SECS), + '-i', slide_track_path, + '-filter_complex', filter_graph, + '-map', '[out]', '-map', '0:a?', + *enc_args, '-c:a', 'aac', '-b:a', '192k', + '-r', '25', '-shortest', + chunk_path, + ] + self.ffmpeg.run(cmd, description=f'composite chunk {ci+1}/{n_chunks}') + chunk_paths.append(chunk_path) + logging.info(f'Composite chunk {ci+1}/{n_chunks} done') + + chunk_list = os.path.join(chunks_dir, 'concat.txt') + with open(chunk_list, 'w') as f: + for cp in chunk_paths: + f.write(f"file '{os.path.abspath(cp)}'\n") + self.ffmpeg.run( + ['ffmpeg', '-y', '-v', 'error', + '-f', 'concat', '-safe', '0', '-i', chunk_list, + '-c', 'copy', output_path], + description='concat composite chunks', + ) + shutil.rmtree(chunks_dir, ignore_errors=True) + + logging.info('Slide compositing complete') + slides_dir = os.path.join(tmp_dir, 'slide_segments') + shutil.rmtree(slides_dir, ignore_errors=True) + return output_path + + def _build_slide_track(self, slide_events, tmp_dir, total_duration): + slides_dir = os.path.join(tmp_dir, 'slide_segments') + os.makedirs(slides_dir, exist_ok=True) + + slide_segments = [] + for i, se in enumerate(slide_events): + t_start = se['time'] + t_end = slide_events[i + 1]['time'] if i + 1 < len(slide_events) else total_duration + duration = t_end - t_start + if duration <= 0: + continue + + seg_path = os.path.join(slides_dir, f'seg_{i}.mp4') + n_frames = max(1, int(duration)) + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-loop', '1', '-framerate', '1', + '-i', se['local_path'], + '-vf', f'scale={self.SLIDE_W}:{self.CANVAS_H}:' + f'force_original_aspect_ratio=decrease,' + f'pad={self.SLIDE_W}:{self.CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white', + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-r', '1', '-frames:v', str(n_frames), + seg_path, + ], + description=f'slide segment {i+1}/{len(slide_events)} ' + f'({n_frames} frames, {duration:.0f}s)', + ) + slide_segments.append(seg_path) + + first_time = slide_events[0]['time'] if slide_events else 0 + if first_time > 0.5: + leader_path = os.path.join(slides_dir, 'leader.mp4') + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', + f'color=c=black:s={self.SLIDE_W}x{self.CANVAS_H}:d={first_time}:r=1', + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + leader_path, + ], + description='slide leader (black)', + ) + slide_segments.insert(0, leader_path) + + slide_track_path = os.path.join(tmp_dir, 'slide_track.mp4') + concat_list = os.path.join(slides_dir, 'concat.txt') + with open(concat_list, 'w') as f: + for seg in slide_segments: + f.write(f"file '{os.path.abspath(seg)}'\n") + + self.ffmpeg.run( + ['ffmpeg', '-y', '-v', 'error', + '-f', 'concat', '-safe', '0', '-i', concat_list, + '-c', 'copy', slide_track_path], + description='concat slide track', + ) + logging.info('Slide track created') + return slide_track_path diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..9573d48 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest>=7.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-311.pyc b/tests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..70d04f33542864c0176377a23830ac6b635bba82 GIT binary patch literal 147 zcmZ3^%ge<81QE+_WP<3&AOZ#$p^VRLK*n^26oz01O-8?!3`I;p{%4TnFE#z5{QMIA z+>+v)%)IQ>BHgt7qHO(=)Z&t2{rLFIyv&mLc)fzkUmP~M`6;D2sdh!IKy4s{i}``X R2WCb_#t#fIqKFwN1_0@>Al(1} literal 0 HcmV?d00001 diff --git a/tests/__pycache__/conftest.cpython-311-pytest-9.0.3.pyc b/tests/__pycache__/conftest.cpython-311-pytest-9.0.3.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8fd76a21a4ca9ffb42121b706d200b74aef898eb GIT binary patch literal 2701 zcmbVN%}*Og6rWx1+Pmuy3;_a!q@-yPCIV7Q(iYWG+Cw6>3atu_D(7Gs&)96Rz3%K9 zg5^j)a7d&a$O)t<66%2#C5QY8JyktA5(jIgNRg^Y<>rW-P!D}?);2gam+s8`-kbMk zcIJJ|%&+0FLEtOw)Xg!KkiW3g`s5~a@DDJXgc2&{NJ0`+&im4`MDj{Pl{}3<;TM`f zA|R-i&;->Jx}ZkF04nE#`A{Mx=J;~qd?XPOyprq6M-x$rFyl^lYXekWCp*x2+7ggN zPlw)rLVv15A2^}!?a*r{^nEl+!|PI_pLWwOpr@%cPhvg$xP!S}$qik)`1J zS07%jDT5pZ|94j$Q-wp85FSLpfAKJY>n{oi@F;XJdhK=lAH*NVYs!GPZY!Z+1W}*~ zZ+r+$r#C93BJIG~r|t3c#9KQ6V`H)~PpoC*h>;O*6PoS$-sU|Z?T2O~gXRxzjc$*= z2*&n;v6>P)=>*)&1!xBp;9DZSv!OffP<00C9;uL9WK)uegI;NVchD^HRp7)tlRHhp zY=?~U%`LPkL~Yzq z)9iqgFD9wU`4Gs?K(jvw_!|mXu0{8>ffw4)o;Fnds%`|UH?X`^jP;uv=eK8S%Ge8K zY)=^j#;d|N`)I+!{xw)-zX5Fem~{kWECROy%qGc_dum1IAu3Us`lwQo9+M|BRjHo_ zvVnW*@iag3Bh{!*4H|6D3RQfM2@O{izzF_8r@zHa)pf9_D8$hc}QwURM`HOs@cSzz%DXFr!%A-xu#nYlDeOFZS6Rv~_< zOr&R1t}m5Jm-PvI$>MmFM1sNOHE1iYDxy;krLCOBMq-4USjEGP#AV; z5spxr*|zKBr9#Z_D#etuvMIzdt+v+cWbq-hwra$DYbLo{q`dO)aSQ6`n%>)z=qGK z&%QEx8!FI-pBT|bz~g$(w79dQEpF`77I*mzkJHG&@b*k2=vV!32*6u}MkjbVTZB|d zx6=WP!H(L68;(T1Fp6RZ=hoqY&UeRxRz}929w1#@m{5Dn{hmalbo$`(_yaO7P*0z%z()v zeN)_(Et{XlB_ssMX!odC*i6nb<$mH)jH{_ ziqh7;BC%R$t&_1@XRVWoT4#lb%G&z5hAc@#4Fd4SN91TrptY5Qs2n_|9BdpZ$5A7&-72ZGoEk()lpJdl*Wjltw3M5%FB`a=Z*flJ)DC)X3QXsl)Y3@>`%s5Jp`!GgBry-_*nFoTaTnbA;O{o3iOnlTjyZt)c0nVqur&b$Zjt! zvZ5Z}y!qan+1Y(RGyGF3rAu(VzrLOSyDCZlz()I0Jm$sEfq5cXk|h_UIhlPcMK!C) zQc;_W$h;>y7iB%Mxfpyag?KSBmw-O?*V3GBX+V=!1Za;H1)8#AKzpq?&^}APBN>S= zakyKC?Dovg7AyArPphSpO|mkIy(rJrSVu2~T)TGR3}Yek&?_6#H` zU`gC{EfTa8Z2bz>_@P{v;a`QdmKNAMhBjYu^<(&)zFB?#G!|ZK^(_f}D-Le86V211 zQrR)&rSG1*P;eFn=`t&CY-v`HcJXrD2Sq^n!DV`m+6zEXAj zLUVNWYn4ZiO&v2s+khdt-R-lu?sE zoYBf7_as}9-DNYW?NBpU$jnpnZLo+|>E9p*Elb;){<(H+Q#;nw23p!cLmSvm_P3I! z8`{XnM}ahwr{Oy}f-L)NX(O#R?+F=18)>$CI7Bo7#U(3KLe?P=AQ+INIOOA_LI)tg zdtCSE}Q4}|`Xl=>f*2BAdWwKNFfpfrFaF4B3_iv8Eqxl&iXWU?Xe zC^It<+b{1E+e1Mb63GnDGHHYIz!=Hp33UvW#Nj0+nB6dEq?i1OWQr$}DMKeIMD-%+ zLvjcS7AoX0k|RitA~}ZSI1n!Yld~uoL~;^{KekT#8<%~7mb`(;ju4u+>|&+t*d>Qt z06-7?)2VJDwC8ZE=gjiW?!k2!!ra=so5|sZ=6$xb;Z~dXga`q)+3w*G(S$B8!LZ{g zcL)TC#nk{xC<)BkyF$hd`2+ysAAktnF98j%1VU~`7z^Q{NxlQ?w|8**-7D1hP*ciK zWkR@ElvrSS;)q}Z)p>gt1A7uy(W{6I2bO?_(5}6{a|;_V1I)DSs)d{uoZF8Rd>H3B zx8l|6LFYzi@UDP0bSicos8$E(cF={qD?9^m{dhfEQ)|(CY`0U7)nfEtf$?_NIZ=zj z&Phjfc%7}}0mI{ofNimY4*I%x1@{~zs3F#}y9TUk($Wq%pWu@c^9irht31)b9g6wZ z)Zl$}K}CZotLGz)y&Z8YW%btL-cByzeEr*z2wDoZyq%m1JGodL_V=)t=)1L}jbyJZ zZpf~ZFKx(6hcO9^8+0gNVzvJz4B4=W*ibN>k0ZGh*c`wvg5ZwuOMGQuW1?KeTyhc3 zfG*#Z0JFO(x!{dpKoYsilL*TT1-oQuJbPROQ)C>;IUs=)GR;%Sw4uBG^R{CyLKH8X zFu}5A6JIW2$s?3dL@ptdI9Ht|E;0%{&m6q0zvypF_;X5{d@F>VBqu;Z&I>Uw`QN6v zBD~5+$`g>mYWR*x!ex65q`o-?2X3}|pA zj)4gYV}AS>$=((E{`3daAY8oS6( z69~1HX%6DHSMLiIKmgB25R^**%i$c;ZB2w0F#BXv2VRMTmipghQ*a`71tRY&XH&0K zCc|KhyoF>035E_bhGYT>I=zu_k9WFwE<>Gz2B&tR%ZyK>-M5ikL4xt8qp%(A5?650 zy@Ja%FoI!=xWhj{9fj#>! zGE^vcL4`^iE8*=|BbII@YZ0$#i5}qiIaL1aS+ponv}oR*wdWSg+)MEhIBcYNXczMs zC}WbAO|v9l!NWX)&yJi10wu}BKHFQKf-wQ_?rhodD6wg^;1FCJl0<^LM5sMfQZ2dq zuA0xeLq$hu7aQY6$4#`wcCoC0z^yEEZgL7Ziod1$4kLP|rxOSy6IQr)wtjsl2v#r$Z@+>CCc{T8#pQ9k4|eoWZRA;&BM=Ve}s7PsTzzG>}NFi zDXmj_M>;EwOH_NHJXGJ49>}uv9@s}bl;hehI6%RPFDWDWrDJ2Ia>;&}r+7wT=JEx5 zv{*^I>fMY*ZUD%LxKVyq?nX%_Pi^Yzw0gHf%J_=7hi|~yxozF%O?Ko>W>HOMLr`20 zuQVMH?^h0fDyS0Mzx+wtBH z^~ZYS$UFS=*K;d+Gk&`jzuk!6ey;8UIsBhY^(&tNzp-xo%a<>^b@}BKo$wDQuSeC- zqS|#m_E|~>Y8-Oo?OzVqVUv5p$FL>UdOJx8>>#8$cf`RwVmyziv@Zxkh(|#Uj*5t9ohQY zA??I21-Od>)h=ADZmYu{bn@145BXE1Kn4r~0Tw7wAU6Z6`9uReZ`}ZX_+kw33@?-N-MCZujsj3ReiOlgrTQuLLVl zW(#{D9r&7}Xp3u2Nz%n)ND^$x4z=K9T`Goc*$%H$M-&xXF>6*Bn`6fswnz^>WGgG8 zE+cwH+5<-(z~z>Ot(eCP!kS-y50nRf!KPqd=4>lokuCE5B0MhA36~S~FlVuelz59B zP%ZxGfRfK;E6k&GhE12~P!#tGXRG z$Cn{2H!LHWNp0mfEVVd_*t>Bvt`>)8SBy@P!uw$-RM@(WWCF=uB=?X^pZ2Cq z(;yaYDv`h%g{?$lUnLkxFm5fttOg~_1s^Pn)r`peo$+~%YS6QA~w zcbwk1Me=#;R@O2z>D;>wc&;~yz{Fgt+$~K*5t9ur$B1JYJ-(eSt%&0kEwRJ`uK%5hAce~<*|mx zyGjipz}CGI2JuBGVw%!jUw1~~gP;!##URihYL-bgZ8#`aANvMb`#^&ylrp<+ZLg2P zT0wkU{sg=yTnxJCg|+5T+A=s_47#m@F8+4HpgRU%Y~)4X5q!hxKteR+aM&mBAn8Z) z9uR&IARmCrxzg?hx1Q$7O(>;?MzMMv$VH%n6D|T32K(mB@4IIQIAQYwl}77-5U5w% z1?ozEgOFR$$|bnSMWkNLV*|1II`dduLd2z4I7?*X>Kl*L^rk`XfcYPsBU1kX-Fagx zF7ezeE{U$VLTy)E@>^SR$=s@?IZb%`E3UA=;!^a;e|yE%vcisbTwz~4@4y()A$I<~ z753YF-m#9)TfKO{oAZ(X-{(ztRJ8e5J21N^ILgJv?O(s%CW^6X$v_t6c)DQ8;>I#ypNUkAy7s-#13?S*Wu!zz58Z|WqIILwZ!td+Zje_0j znmklq08*MRO_vuwpWB@Sp+Yrtm4z4D5QywNl!qD~&om`K09W@47{V8!h-q+#z^!{m z(F9F15JQv~f>LQNC{^4t)(I$Kj8B;c>@ZWHXSEsiVVxM*Ke=$rS?9QJ87M!h$pbW6 z9(_KuI|D)`fts00;)ON<96Jx?friI34I+T6dj$;Pi%`Tg7%gtyGm0h%q6I+=QGOJZ zN;5&JlJL#3)`133C}mceDTV>d!CFClTb?b_@lEy)+!DYk?a5_$$-iepDOrJjn7T>~sbagKZxmUt-`jS!JhxU`~5C!GEB z+sbBQJX^R6-xKsZN^?J;p&KEPC64MFgeI&Aj%llI#C}L7flEJu?ZEA&K-O)_OYi~X zCioza$I^hne0LhKkg}er#c|_V8ZuGtRCbbP0UarCIw8mn#>1ow8YcKA5`29y@i&QN zDlVRib1-dQ#cA`K=*OyIndxnV>&Va?OD;mzGOo0Ah0I{5X0cP4cf7sLr6VEZN!1ha zOEeux?lU0Vvm>wN(>a3gxD&%{8>Z+DM`4Z#%)sfayv;P+iMV|wIAxFbkU=yRv9g7v zo+d26C~YPjX>)@70*c6WBtr-aq#f@ufu#_+F=A;()Xgv^IO&dw&%#SEvTv4Q zXr>V9O(h|@kbG(^EvK?3nTI;VaGC#v&}vy3yo_nPSH2whVxYWQc~*~&He#bYAGKGi zUAs5Fy!pk=^3$qXSH~La*v`Y3a)0@wYF|wrsmmh`c?6g?sv|l2r}@w3%fG4gl|QL0 zRfj9TtM$KMHUIMPdH?g}x_rMO->=E{UuxaEQ#E<8G=)o(!O|S8I!5Z+AkA@NZWEO3 zJd_6;GNc)i1~q@4@O2ClPymkUN>dEvO0eK?&2+K~o!}t>g>RLnJ)8Z38mx#J1)kaP zErF)xbZ>tIZvO%JvGkzypmL|KU2kaDL97k~scF}%!!I;+Go0Y^L(l6><3U&I@*yK^nc{+uw*23PpQz&J6U5*!y|zRHQXI<{V?BTA2- z)*r{A-2LkVF30hI7P%GV=uks5_%zW+b0~Csn&4uczL)zf>qlSW)96e#uWw|GIr13F zVaylI=Rl4_q9`8WtJ~od+o7K$AztIpBjKkt{ybJB@#eAM6V>Oa?t4U8eD7HBNl#4t L$+2*jP*?v8IV$>f literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_prober.cpython-311-pytest-9.0.3.pyc b/tests/__pycache__/test_prober.cpython-311-pytest-9.0.3.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d51b7f647f64b9fe95017bf4827dff0ba6722bd7 GIT binary patch literal 17714 zcmeHOO>7iNmd>im`c1oRV*?EsZ1dAlm;sxAW(;c@=$_F^u)8ojYWAm8O_ZUa>~cqz zp_|RpnANOQgBZ1MU5D1n+5o={>aA%YH8+D>5U? z1s;B8XLd-ZU%hzo;$>uH=8N}Uy!hAN-jspk;r*4uT+%T9g9_~<>>}TmfqZT-gN2I5 z<&gLd&ql6kgp2Xn#O1_n@^TXN$j^++DP{s%EDG4eVt~CY4wz;M zz&@4)>}M&!0cHWNV?BU_EOgPx^gf|3oX>l!n6Dx#7 zDPG>FE^KI5?P=4UxPDc=^pAM&J_%J!j2y;sQ@fE-NtCy!g5f0eR+S`6R#HyVN!^GN zSH*HH``@+ed*hyRER4HF+`(~YJ!>4%O`2>qqUFtAAN3w`y@eXv?kzCNJ=#@No8aAd z;Y#W~=)#rKxmfQPrg)M)EY13y9{0KSuHm`%>XP)5`&<)vwn_Y3l@v4Z<@cX|mY}bF z|LpwXR@^T0d~Vjxgj(UkTqbmT-L5_JpH61%(Zbv)p@%Yd%Ph>zl%Odhs3CCZSvqIi zd0rMAN5BE%ktg)wWf8n4kNSTNIg`&ATsQS}}HPW<3P*|P>sN+^AA6YvI z73X7fXS0pFL}CHjXxB&+XsbXXK7EQ%;Bis%C7TPJ4!kEYAL znJwbPT#F*wF|$R>eRQAlxK4K{ju+Sp%BFA(U7rVR1vLW(`N&q#dh2lptO^!;#U2JM zwyr%4@S%sXEou>?yxad!qytYHQXST(v_cLt4+b%g&|B`Rjn z)cMsL@Yk4mK&5XyN_4y{^NzAR74?o3kFDIK&&N5`s$*D1^ zF3QQUhwK`{vpM_lq+%Rg3azyci5e@Y5?CA;K7Ykw<0e*k}uhWnQ{ zV1=^qmk~YwMG>yoW{H}@5*0RUhb8)|cHYZdqTTGgpd}jrKeI&RUG&$>S)vJLi6&X< zja#DD`P+$Gh4QtwDVm}}R;m?JdM{^}iuNzIO9LwO!E=mG*`rrgq=QP43VFg(WA7S{rr3_35N-IvE1 z(*txY`9(wo{t_&O<;8>>_)D%K@E2IB)8)38L=p6h;y;chS6VvjInQZ`pU_eU0)Gj% zz{N7pReDB-v2NN$HcUSVKMPIqzXUN8zCA#pL0VIr62U#Jib)a>mvodCwID4M;WT=j z+(8QP9o3e;C@Of2M6PaW^<2%Dv+XYVyjBs?3xxSpgfEi{M)>}UItF8 zqnX;VcxJ^)*UinXSiq=8!m~6!a8gO|{1tkUTr3lG`KKffJ=1!#3SEY|ate8vyzU~zh7G{wz&HIr2 zkgg$)wdZ5}E!tEflu>|H6sfkLU_pyRWp&HVAcci>0imt}xf$$Ly~BWl2`Hkj3JZ#; zn|BygL3*SI5A-^GdtH6*<5WAKyT)kF)fK-N)l*%AdBCxhHdQv7UP)bDTRwgW0 zWh}hcoJ>%;DigZaMLYMCN)!oJS!5m{<-(_-1Ik%ngXz@Je@9pQdw^DWenx6(T9)Cv z3G4w_RWYwuidpML4*aC!?Vr3zIY)kZrfyC&%!#Hs0c53b{a1bOKI(gSx!ma6*X-MO z=ZA}@7eB6UU+EuQyilE7irhWEc!B=@>B7&%FX2RAyl_WQh%Pb7Q=RnaZCutYMcRCK zlq8ZuwNg6Ow|V4M|3JDyo#|BGYPp*d>fVI%X6Bbs-R4`n`mRKf2p01tf|)F?Eru7h zPua6Nk(34%o}^{-pyJV5(9|W4V0v8_uHbTS%2l>;X0aY$kjn*uMSUrBgsH|>`Qs-nru(I27KHuT)}kI zcU+&^Vw>1z$8vpYElpRG*RLj5qc9CSB{#Bi_zDY zBKQtdgBh%j>$y0AwU)_Wp82;(-JEKeQ%!RUh{)$0eq;^TE;X#}O=~;lbM`I8@13lh zqqVaD)sy(KMuExmu{qjoB?Y`nw&eG>Qp^&e(M;Ss zg_ogr5ukdicJa~k-%W6UQNBs;yLrlAs$KNIWxxY3nxy=W?qb9YmxpjNaMts0_QGEs1hnB67=$UpOn%NTY~5^(=T zfJ+LRstZbff2%-~&SIzQMAI8h6yN?QCW_bZI_Vi2M-azyBC7}OSsn$i)B{7RXJeh+45s3fw)e ztLY!HG>WKUsh~3sq_t3)$xgu0>cX|whz&)t-9wY)#eMH zl)M$15!*Kc^DQSMP|UNdhF~@ZCe&Fw_zx_c+5XMV#P4-sV(wU!++#7jn zgrXz0(<_POXYsr7rQSwjTQjk(p4j#@f(lV9z<&|ne-rR%t6|@L_uXpbZ)gujmpOZ^ zcCK!YK{cO4&K59vJ~qdiZCp|WhY7tw6|$O?w^yp2lO@RxTnY12&{D~k{Qg#fCbDQ# zG~iboP7g+YGiaXP82`=Y5TKUOs#M8&e2A|>mEeNyV-1@j!@8oVMD)#&A^?x*q_nLSArCk6-W0L2uWkjaE<*L(xj_i1jh;C z@Kt*eeCL#VS3H<@9LL0Cg2)Y53m{iV_YGckunt`lz0vW9hx>!WHm7**Z5$*vkgl zkkjjqNpcNivX*Jd!kExKN7;IB90Fq!|I-{3;nQz8e>-+_k`fc~dysygKvfz1Jb^0! zG&_sKMTVw1IpZ03?JR|ZSzv{%M;)ycE(N5fW}=k0}J+0K}9Wrn^| zxnD*&{+Fmo%jcZ>(Ms9n94}|fh1onW5S6;&Sqq#tMP||glR@sI>60!o?Ik4_U0&X2 zJ3N0mC++#|SihOX|G`!i76?{^Xg9rY)m^yi!fB_~cAu$1R>ms3Hl^1tQ;&jOyp1Ll z^{{`7yNBzZwC-*EZ20c*($(50jpUwYa?j#>s}r#;^S$MPy1Aoa?r54jfY46O0BI%N zzqF(JVQpRQVl7$k*}iygB|Y?2dgr6`P9$-s$D8T##Sgx=($!l(pSp*5z;F#q{lnF% z8n!%oTPoEaO0{W{I8ac#DJ6$0qDoR|$D_Lsw1Xgpb!%C@6=>-rDDhPUGDy4>Qp4s* zZ__Doc?AU}1-z0Sbjj~;rI;nMVy#=6y!ZX3V$;G1?_EY>1G2q#EZdJ>kj4)dmDcKJ z!CQDzz8rewCAfL_l943)&X>rc|D)=xyjS5yW`P0qv}7rOA@F_g^W^%@8K^0!L8H5L=zarZuszL!>8*ygyD^Fg!t9IT`QaU zq>eoaH&ssO)0{WlOnD$bzO27UxfJ)by0#;G_rRHZiMn1-?CN_) z-SGbACnZ$d!4dh#*6YTmZgb>j;s7qxbNCPY3?JqcpB-r*pZ7PKHx?T^B`oUN0j%e_ zToa4q@XYw99aqt9BQ|1$)Z-(511b?m+2#}D5v*#@w3BwKZkBW|CH-?^1)3HcNuG5* z86PPw{SJKmg}$f5zX871cK8jCWA^C%G+w8^_vvcvoYni5xSBnBKhyWV@SLWd+53ZH zj2^u|(D%OZ+$(s0NQ}{=_p^QP3(viR_lLz8J$m2jdtZ3&tlqaZ2-gN7QhS*ydDE4u zpVJ>*n_H+oyq5E_Uxv)|&1vMW=DfwtO$^?$d((vG`s+C_&}%s(FaU$(8j|aehKru( zFrQ;q0Rzg_ZyCPE!@!Q=!D{)GMR^o5dfn`(6e7x2)ry9#Z_SfKGO#w&k$ed~ko-`a`P92I4LP!xn zq}gARx(-DIS|bo48q3lI5Wv%nl9C3n1SbqEPqmU@V!rn$ccW>}G(G{ce*+HI3~?p_3uJ#44%Q@a)Q-%_mf%Ul5$5Ef=pcj?0YniF4P(7ruQ#jRxKd5iE1i(-Q@KaKz7OaWey8AvW{{9OO5J0yxyXYE&293 z3e;OOQ~q4E~1$-%uNJ;U#vNc!&gP8#pB{=E~sKJr?%-`lU@JMcfd zcX)1-x%*5H5+x5(h;0nemqlI!OqkbvL#iX9;Ot&3>%l?R#2~69f3SZM6Bbo^GNXf%f^tr ziR1#3w~&k@xrpQkK%S-fDY7Y4%m7hi{#8}DuEzZDQVK)az|xY3cO7K_(`*(;`4Nxo zh2smm7}S?4JI;ph*ab?G*mVHn3ccUMHqsH;LwLBbf1_nxg5y*DFBv#AtxHccO=|&J z-j2+Lmf%ArhnNe8qK6<-1Qcm-PY#Vc6cGqQMFfTOSeM#tq$n+(>QYBWiXsAKBnWiC z163Poz&@N;oQ`Ux;w6v4S>JEH`;8pP=HX`S9vXcHcW6H}ifuGe5Y-Qj;J$&d^30(T z-`>)9*#e9LTZi4-?nbq*Gwp2bCom&hL~;e^wrDH4go>WFlB;0Rxd}}2O#s)|eFInp zE8K*<58#*4jtu}(=-K_1$2Z~7SbuT{rJIdAO$&0fupODREy0tLBTUGh!VW@65kRED zr+*BhC?XK#(g+GoZnl|7Q5rPXrH*VAMFhGdL7)R3sM<&a_A~@xU=^yBif`ll53zoJ zr@U$K-^=gX9%7rlJ;X#Y{&haaCSKEHY?@+O{&PY12idzY6GqV8A7oIXp5I@4d>amp z;*)KZZa21@);w_3j?DR%;7P*~=KP`PAcPbFL>hcv^AcPbFL>k;~L)#8T z1cG035ELSAwV6m!8ZiaCbs^qP2 z^%*&jhbXy=K}NjF!3(az#R9$ju}HOHcp+nvIlpBkA712;svsTec)2zhm2XTe%GV+2 zm6A82p0I9e!mS@DYWl`9d@DIPAE8C@mEq54!En@p2|2@xtvUz=;0&ue!AOMV-$8~i zqVo4Jc+Hk!-p(-Q<=@0-8utUQ`@YQlyj-<+UFR0N1O52IvxmPpo*25WA7hf={jaGl zwzXp|*NnGgZJ`-&$J#s1cRK34QKFlO&ta RFQXWetf9{%xSepP{2MJt(MY~=b(b7|?TV6Z$*x@^5)&z?Da*3HWw)~I)OHR zEOiM8MGmUrHoDn|px{Zd;rxQ0iv9x9pb=q#o(cr;r4SIH2c7!f%l+({V(pk~AX>P8%3pOFBVHj)7Q4fVE?O})ZtZe>+> zpp;)XbGB*Oxep44Sy}NJK!OxJQFy)_12!KkHsZW%P4!FlU=JUuII5xULCbT|T8lVJ z9Y^c9oQM%Iq6XO`Ko1YU)E@KIqE2*8P}N98aflPy1Ad+ho@y~CW_@AAjJU1)<0;Bo z+%NAz;kjQ?K2WSP_JF7=Ye~NxT(LX@m3!b$v8#8FvI^V;XAymzs62KLDk4)Ei6gFU zNE%P9Excz`rfYK-+FMvhLK?A684Gck{3NyodtQs%C;k0V6sfOxJ5F5gE7yJtAHV~( z7KMk@V!=IbpF?&o`uur%oeLvXOE@tnvCd9VE$Jk8RqM3GQ`#TH(46Ew)zFdtUPL~f z-<~Y^YUC61KRj8|`)Z%RgnVj@zE9%p{H2Vv(eI=RDywNnIE|V-l3WX(MmUi%c#^e* zp@3i=xV4qKTgD2*L5AWtJ%cOSZ-E-B=#jK_M#w%p5Fw~rn zf`W&d@ljClP*YiJvYNN=Yqd%9LBX;ut#%H_na^5Tt>#85mP@exSVKHAecii zk6@9aT+XV8>T*`yTK!LPvufH_Vat@*$t<&#O#{=}I7Q7z@vEo!sZ#vbXg>g8cyoGc zYap;a)39KMp2_yp1QOy2bIDqzP>8$}~W%__~U7Atr2#auaGGTl_p%HJ~)?e@RNUQ?k`zDa3? zx`VlU6Zo)2Ab4G!3 z{&nP#FLT-rRAz&f1zzEkUJABc4puJl^>VW5X{k1A;WlU5*Wvk#v{JDzmTaq7D6gCJ z0`yzIi2GtOnDaTFtZur4Y`^4Prdb#fOaaCE100`i<*$1B(c;HT50{=K>Svq!g_eF{ z`^G-$`<$G8LC!w8Q6H^aPp>!0REtbC$P^=Qkdd9mN6Ywcd1v{_SW_Qq=_62Wa0mr5 z@{)|S$VihxZ1OR`pAk4t5DR!1aT|`xM04DsIc%J}55CMCF3Xo9dO}q!k zgm?yI@b;EJ2Ft(NAY(gMcdkB}YwDSno`Gt88bCwO)TdwQV^}1>)!0ii)?#Ij@kw|i zZ{Z|BD+oCTx5mSDAfP|T;a(w;SGyYOBC4VTa4e4+KnNmOMsV2rwCKYb2N=Aem3QS_ za2jAL*egiiwfrR>Ms4KByTIa&enw;pRRQ~0 z*tI?-{&yl^A;KC0wlqPTcO7N8!ff5YbhiZ!u!F&Zi6eD~1tVK=LVI`E#zWHBZEYK} zN5Os+)*RT{%*A2chv?9LG?J1AfAhIRHfhhfLv%2A(nx(0;Z~`kfo&J}tb&C79psi@ z(n^F)>4fZyPOG36V^%=lt*zvT1^Yf40f&johK+?^JX2i_vAO()QK&pq47S}hP!s3;A3tytI zZ0I?p%`g=bWFxn3npF#wC>Sk;@+$p4km2@POJE>@KGFIfeedImhZDOW*0rW~zNMYt zz8*xTn%a0v8wVsA+`V4E(ID9-$+k!q3j67kpQqn>k$$H>`!w22Pqfk#+wbq|$9Hc( zywV`2`TdfdZuw=7*;&}V4W*{vz$ru(nmA@>LEt(N!LbFE=(&4a;CRmfg7*XnB%;g< z90NUgCr$&*i@}2QbR2e3l79F)nDqMyWXd;C`bFPJ`bED{(k~P*=qdd|bpL(n*Ih@V zhFTL|bu9@In`AiVQA;@~>kFgLNFPx%oRMU-&`LoLWcdsco38q)cMsUFboV|^DALh` zDk*dDm9<{zeZI+i@4;az#9_Y|vC)p84Ugg@bKf}S`^GxzKTjXh36N0l(+dFUMFi6b zn5J38(j0<$1k6*!*d5n%_^yKjz-D$?7iM)FxZnPcJvge2dIC$o#jG6a?#QBx(6;C`sQmYJ?e3 zjJAB1+JmG29FIS{AXN;pk7BG8uMt(j45};#>q0*IJ@9|{`-f;pUo8$YH@?ouAhVXR z9!nBI$QUCHIW!frFhJemiwy)bGHb$0mw96OrtT6WBo9OKOUTBs)eiD&K;pJd8J1ti zyqHnaIBuM9l3vUxW#c#MUD8M*&+CLNY*`n2nWTg#$p&xHWf+ZK1@HL1dWH(e_|Si{7-4+u1Qyc4zA=i%=Ronw&%DXa9U!C znOTT8MPYy}k27I*Xm%?YXM(H}%hP0!b&E66>oB-`Oe7U1t87;KBSiN>hNR8h5e@UK z*3~A*0qKu%)Iic?rXwmf7YtD?Sg2b4s8l8sj7te&n`1#aX&Y}tuXo|Ge*W)>-}>7k zgbjr*GWgFB>p#^bH(LZu`kVV?q9Nb=+R2^szq|10!spu93vH}^yQyVcT6X*T{_&v( z8QPiM9ej8h3jBUahFX4^V~_w!O}~Lth$=L3%+9R9bsz-M?w}yzV*(7`(8_5AADlke zD@YGzE_GpR#U-Lu{{ZY`h=i|+rkc8{)FL3G)bKABYEft*PShcRSWv^psF!!G;rGBU zLv=6`4v}<+bjTkmHsAhjNf-GO99vkPjv{~6Ab)~%Lh6R;K>kE~P^$8XJ*Yy_?H*L2 zs6h{^kmYF*|F;{cLjqB9A=%)U9KluM8)VnNj*^4zrot0}2Y=BKs}W|$C2p}e07$5x zp@drS^WIQ+eJb2Ajg4xtumag&FY^qcaY%&w5%tR;z%Z(vjkP1`EU198ZUMn%Iq^L0 zC7!3VDOV4Se$>!{A4Si=q|EjUimZ8xk|l^)l=8MyQf;P7T>TU?76W|c%u?0fq`!u7 zF_L1PV^V8hA7q);Y&`yiW{n8Jo3rs@?5~ztl zJ0}rUXyTZiO9Iz{5JW7EiV=HG0{pNEc@3rtR_jy@$ma(byrGqk>-F||4E74rcP%qn zlpS!B{*MNjsqdc2@m~x~@`MZM-$`nINDDUnOhBOAgoon?1C1t^S@PE8N~L@cUlYs4 zu)H}-cCZi-MeClTj$yCtnZkHry6G^vRH_&oMRS=N(2gR~`XhjYsH&>_7&Pkfm)d53 z`^tDjz7OI=efK~Spg*RbIZy;R7FS0O6ah}euqwc@gnIHo5nxDF#}AZF!1(_srugjU literal 0 HcmV?d00001 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f0ef0c0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,54 @@ +import os +import shutil +import subprocess +import tempfile +import pytest + +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.segments import SegmentBuilder + + +@pytest.fixture +def ffmpeg(): + return FFmpegRunner() + + +@pytest.fixture +def prober(): + return MediaProber() + + +@pytest.fixture +def segments(ffmpeg, prober): + return SegmentBuilder(ffmpeg, prober) + + +@pytest.fixture +def tmp_dir(): + d = tempfile.mkdtemp() + yield d + shutil.rmtree(d, ignore_errors=True) + + +def make_test_video(path, duration=3, with_audio=True, width=320, height=240): + audio_args = [] + if with_audio: + audio_args = ['-f', 'lavfi', '-i', f'sine=frequency=440:duration={duration}', + '-c:a', 'aac', '-shortest'] + subprocess.run( + ['ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', f'color=red:s={width}x{height}:d={duration}:r=25', + *audio_args, + '-c:v', 'libx264', '-preset', 'ultrafast', path], + capture_output=True, check=True, + ) + + +def make_test_audio(path, duration=3, frequency=880): + subprocess.run( + ['ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', f'sine=frequency={frequency}:duration={duration}', + '-c:a', 'aac', path], + capture_output=True, check=True, + ) diff --git a/tests/test_ffmpeg.py b/tests/test_ffmpeg.py new file mode 100644 index 0000000..bdb35e2 --- /dev/null +++ b/tests/test_ffmpeg.py @@ -0,0 +1,37 @@ +import subprocess +import pytest +from mtslinker.ffmpeg import FFmpegRunner + + +def test_detect_gpu_sets_flags(ffmpeg): + ffmpeg.detect_gpu() + assert ffmpeg.nvenc_available is not None + assert ffmpeg.cuda_overlay_available is not None + + +def test_detect_gpu_idempotent(ffmpeg): + ffmpeg.detect_gpu() + first = ffmpeg.nvenc_available + ffmpeg.detect_gpu() + assert ffmpeg.nvenc_available == first + + +def test_get_video_encoder_returns_codec_flag(ffmpeg): + enc = ffmpeg.get_video_encoder() + assert '-c:v' in enc + assert len(enc) >= 2 + + +def test_get_video_encoder_fast_returns_codec_flag(ffmpeg): + enc = ffmpeg.get_video_encoder_fast() + assert '-c:v' in enc + + +def test_run_success(ffmpeg): + result = ffmpeg.run(['ffmpeg', '-version'], description='version') + assert result.returncode == 0 + + +def test_run_failure_raises(ffmpeg): + with pytest.raises(subprocess.CalledProcessError): + ffmpeg.run(['ffmpeg', '-i', '/nonexistent_file.mp4'], description='bad') diff --git a/tests/test_grid.py b/tests/test_grid.py new file mode 100644 index 0000000..e6e2c5d --- /dev/null +++ b/tests/test_grid.py @@ -0,0 +1,42 @@ +import os +from mtslinker.grid import GridCompositor +from tests.conftest import make_test_video + + +def test_compute_layout_single(ffmpeg): + grid = GridCompositor(ffmpeg) + assert grid.compute_layout(1) == (1, 1) + + +def test_compute_layout_two(ffmpeg): + grid = GridCompositor(ffmpeg) + assert grid.compute_layout(2) == (2, 1) + + +def test_compute_layout_four(ffmpeg): + grid = GridCompositor(ffmpeg) + assert grid.compute_layout(4) == (2, 2) + + +def test_compute_layout_five(ffmpeg): + grid = GridCompositor(ffmpeg) + assert grid.compute_layout(5) == (3, 2) + + +def test_even(ffmpeg): + grid = GridCompositor(ffmpeg) + assert grid.even(640) == 640 + assert grid.even(641) == 640 + assert grid.even(1) == 0 + + +def test_composite_two_webcams(ffmpeg, tmp_dir): + grid = GridCompositor(ffmpeg) + v1 = os.path.join(tmp_dir, 'v1.mp4') + v2 = os.path.join(tmp_dir, 'v2.mp4') + out = os.path.join(tmp_dir, 'grid.mp4') + make_test_video(v1, duration=2) + make_test_video(v2, duration=2) + grid.composite([(v1, 0), (v2, 0)], 2.0, out, 640, 360) + assert os.path.exists(out) + assert os.path.getsize(out) > 0 diff --git a/tests/test_prober.py b/tests/test_prober.py new file mode 100644 index 0000000..94f4722 --- /dev/null +++ b/tests/test_prober.py @@ -0,0 +1,81 @@ +import os +from tests.conftest import make_test_video, make_test_audio + + +def test_probe_streams_returns_streams(prober, tmp_dir): + path = os.path.join(tmp_dir, 'test.mp4') + make_test_video(path) + info = prober.probe_streams(path) + assert 'streams' in info + assert len(info['streams']) >= 1 + + +def test_has_video_true_for_video(prober, tmp_dir): + path = os.path.join(tmp_dir, 'vid.mp4') + make_test_video(path) + assert prober.has_video(path) is True + + +def test_has_video_false_for_audio(prober, tmp_dir): + path = os.path.join(tmp_dir, 'aud.m4a') + make_test_audio(path) + assert prober.has_video(path) is False + + +def test_has_audio_true(prober, tmp_dir): + path = os.path.join(tmp_dir, 'with.mp4') + make_test_video(path, with_audio=True) + assert prober.has_audio(path) is True + + +def test_has_audio_false(prober, tmp_dir): + path = os.path.join(tmp_dir, 'without.mp4') + make_test_video(path, with_audio=False) + assert prober.has_audio(path) is False + + +def test_get_duration(prober, tmp_dir): + path = os.path.join(tmp_dir, 'test.mp4') + make_test_video(path, duration=5) + dur = prober.get_duration(path) + assert 4.5 < dur < 5.5 + + +def test_get_video_params(prober, tmp_dir): + path = os.path.join(tmp_dir, 'test.mp4') + make_test_video(path, width=640, height=480) + w, h, pf = prober.get_video_params(path) + assert w == 640 + assert h == 480 + + +def test_probe_file(prober, tmp_dir): + path = os.path.join(tmp_dir, 'test.mp4') + make_test_video(path, duration=3, width=320, height=240) + info = prober.probe_file(path) + assert info['valid'] is True + assert info['has_video'] is True + assert info['has_audio'] is True + assert info['width'] == 320 + assert info['height'] == 240 + assert 2.5 < info['duration'] < 3.5 + + +def test_probe_file_invalid(prober, tmp_dir): + path = os.path.join(tmp_dir, 'bad.mp4') + with open(path, 'w') as f: + f.write('not a video') + info = prober.probe_file(path) + assert info['valid'] is False + + +def test_probe_all_files(prober, tmp_dir): + p1 = os.path.join(tmp_dir, 'v1.mp4') + p2 = os.path.join(tmp_dir, 'v2.mp4') + make_test_video(p1, duration=2) + make_test_video(p2, duration=3) + items = [(p1, 0.0), (p2, 5.0)] + results = prober.probe_all_files(items) + assert len(results) == 2 + assert results[0]['start_time'] == 0.0 + assert results[1]['start_time'] == 5.0 diff --git a/tests/test_processor.py b/tests/test_processor.py new file mode 100644 index 0000000..45f187b --- /dev/null +++ b/tests/test_processor.py @@ -0,0 +1,39 @@ +from mtslinker.processor import VideoProcessor +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.segments import SegmentBuilder +from mtslinker.grid import GridCompositor +from mtslinker.slides import SlideCompositor +from mtslinker.audio import AudioMerger + + +def test_processor_composes_all_classes(): + p = VideoProcessor() + assert p.ffmpeg is not None + assert p.prober is not None + assert p.segments is not None + assert p.grid is not None + assert p.slides is not None + assert p.audio is not None + + +def test_segments_uses_injected_ffmpeg(): + p = VideoProcessor() + assert p.segments.ffmpeg is p.ffmpeg + assert p.segments.prober is p.prober + + +def test_grid_uses_injected_ffmpeg(): + p = VideoProcessor() + assert p.grid.ffmpeg is p.ffmpeg + + +def test_slides_uses_injected_ffmpeg(): + p = VideoProcessor() + assert p.slides.ffmpeg is p.ffmpeg + + +def test_audio_uses_injected_deps(): + p = VideoProcessor() + assert p.audio.ffmpeg is p.ffmpeg + assert p.audio.prober is p.prober diff --git a/tests/test_segments.py b/tests/test_segments.py new file mode 100644 index 0000000..01c3b77 --- /dev/null +++ b/tests/test_segments.py @@ -0,0 +1,48 @@ +import os +from tests.conftest import make_test_video + + +def test_generate_black(segments, tmp_dir): + path = os.path.join(tmp_dir, 'black.mp4') + segments.generate_black(path, 2.0, 320, 240) + assert os.path.exists(path) + assert os.path.getsize(path) > 0 + + +def test_ensure_audio_keeps_existing(segments, tmp_dir): + vid = os.path.join(tmp_dir, 'with.mp4') + out = os.path.join(tmp_dir, 'out.mp4') + make_test_video(vid, with_audio=True) + result = segments.ensure_audio(vid, out) + assert result == vid + + +def test_ensure_audio_adds_silent(segments, tmp_dir): + vid = os.path.join(tmp_dir, 'without.mp4') + out = os.path.join(tmp_dir, 'out.mp4') + make_test_video(vid, with_audio=False) + result = segments.ensure_audio(vid, out) + assert result == out + assert os.path.exists(out) + + +def test_normalize(segments, tmp_dir): + vid = os.path.join(tmp_dir, 'input.mp4') + out = os.path.join(tmp_dir, 'norm.mp4') + make_test_video(vid) + segments.normalize(vid, out, 640, 360, 'yuv420p') + assert os.path.exists(out) + + +def test_normalize_with_max_duration(segments, tmp_dir): + vid = os.path.join(tmp_dir, 'input.mp4') + out = os.path.join(tmp_dir, 'norm.mp4') + make_test_video(vid, duration=5) + segments.normalize(vid, out, 640, 360, 'yuv420p', max_duration=2) + from mtslinker.prober import MediaProber + dur = MediaProber().get_duration(out) + assert dur < 3.0 + + +def test_deduplicate_empty(segments): + assert segments.deduplicate([]) == [] From 2772f7dd129218dbb3cec9d71df73a414ad59c7b Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 10 Apr 2026 07:53:10 +0000 Subject: [PATCH 50/65] Add __pycache__ to gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ccb6a9f..232bef8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,4 @@ *.mp3 !mtslinker !get_sessionId.mp4 -!setup.py \ No newline at end of file +!setup.py__pycache__/ From d9f9b0ddcf565e8d41fa7333b6190344c1118c0b Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 10 Apr 2026 10:39:32 +0000 Subject: [PATCH 51/65] =?UTF-8?q?Remove=20webcam=20audio=20extraction=20fo?= =?UTF-8?q?r=20grid=20=E2=80=94=20causes=20echo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grid composite already includes audio from input 0 (sorted so audio-bearing webcam is first). Extracting the same webcam audio again for _merge_audio_tracks caused the voice to play twice. Audio-only tracks already cover other participants. --- mtslinker/processor.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index e01df25..25da6cb 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -180,12 +180,9 @@ def analyze(self, total_duration, downloaded_files, directory, 'sources': gw['sources'], }) - # Extract audio from ALL video files with audio for mixing - for f in video_files: - if f['has_audio']: - extra_audio.append(f) - if extra_audio: - logging.info(f'Grid: {len(extra_audio)} webcam audio tracks to extract') + # Grid composite already includes audio from input 0 (sorted + # by has_audio). Don't extract webcam audio — it would echo. + # Audio-only tracks cover other participants. else: vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), From 8be14737cad1941ed23a0905a9543c25d7ee046d Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 10 Apr 2026 12:01:27 +0000 Subject: [PATCH 52/65] Extract audio from non-primary grid webcams, add audio pipeline tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grid input 0 already has audio — but other webcams' audio was lost. Now extracts audio from all webcam files EXCEPT the one used as input 0 in each grid segment. This captures all voices without echo. Added test_audio_pipeline.py with integration tests: - Grid audio no echo (same source not duplicated) - Grid takes audio from first input - Audio merge preserves timing with adelay - Segment duration matches plan - Black segments have silent audio stream --- mtslinker/processor.py | 20 ++++- tests/test_audio_pipeline.py | 139 +++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 tests/test_audio_pipeline.py diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 25da6cb..de259ab 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -180,9 +180,23 @@ def analyze(self, total_duration, downloaded_files, directory, 'sources': gw['sources'], }) - # Grid composite already includes audio from input 0 (sorted - # by has_audio). Don't extract webcam audio — it would echo. - # Audio-only tracks cover other participants. + # Grid composite includes audio from input 0 (sorted by has_audio). + # Extract audio from OTHER webcams (not input 0) to avoid echo + # but still capture all voices. + grid_input0_paths = set() + for seg in segments: + if seg['type'] == 'grid' and seg.get('sources'): + # Input 0 = first source sorted by has_audio (same as execute) + sorted_sources = sorted(seg['sources'], + key=lambda s: not s.get('has_audio', False)) + if sorted_sources: + grid_input0_paths.add(sorted_sources[0]['path']) + + for f in video_files: + if f['has_audio'] and f['path'] not in grid_input0_paths: + extra_audio.append(f) + if extra_audio: + logging.info(f'Grid: {len(extra_audio)} non-primary webcam audio to extract') else: vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), diff --git a/tests/test_audio_pipeline.py b/tests/test_audio_pipeline.py new file mode 100644 index 0000000..0884c9b --- /dev/null +++ b/tests/test_audio_pipeline.py @@ -0,0 +1,139 @@ +"""Integration tests for audio pipeline — catches echo, silence, timeline issues.""" +import os +import subprocess +from tests.conftest import make_test_video, make_test_audio +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.audio import AudioMerger +from mtslinker.grid import GridCompositor +from mtslinker.segments import SegmentBuilder + + +def _get_volume(path): + """Get mean volume in dB from a media file.""" + result = subprocess.run( + ['ffmpeg', '-i', path, '-af', 'volumedetect', '-f', 'null', '-'], + capture_output=True, text=True, + ) + for line in result.stderr.splitlines(): + if 'mean_volume' in line: + return float(line.split('mean_volume:')[1].strip().split()[0]) + return None + + +def test_grid_audio_no_echo(ffmpeg, prober, tmp_dir): + """Grid composite should not cause echo when audio is also merged.""" + # Create two webcams: v1 has audio (440Hz), v2 has no audio + v1 = os.path.join(tmp_dir, 'cam1.mp4') + v2 = os.path.join(tmp_dir, 'cam2.mp4') + make_test_video(v1, duration=3, with_audio=True) + make_test_video(v2, duration=3, with_audio=False) + + # Grid composite — should include v1's audio (input 0, sorted by has_audio) + grid = GridCompositor(ffmpeg) + grid_out = os.path.join(tmp_dir, 'grid.mp4') + grid.composite([(v1, 0), (v2, 0)], 3.0, grid_out, 640, 360) + + # Verify grid has audio + assert prober.has_audio(grid_out) + grid_vol = _get_volume(grid_out) + assert grid_vol is not None + assert grid_vol > -50 # should be audible + + # Now merge an audio-only track on top (different frequency) + audio_track = os.path.join(tmp_dir, 'extra.m4a') + make_test_audio(audio_track, duration=3, frequency=880) + + merger = AudioMerger(ffmpeg, prober) + final = os.path.join(tmp_dir, 'final.mp4') + merger.merge(grid_out, [(audio_track, 0.0)], tmp_dir, final, total_duration=3) + + final_vol = _get_volume(final) + assert final_vol is not None + # Final should be louder than grid alone (added extra audio) + # but NOT double the grid audio (no echo) + # If echo: grid_vol ~= -21, final_vol ~= -15 (6dB louder = doubled) + # Without echo: grid_vol ~= -21, final_vol ~= -18 (mixed with different freq) + assert final_vol > grid_vol - 1 # at least as loud + + +def test_grid_audio_from_first_input(ffmpeg, prober, tmp_dir): + """Grid should take audio from first input (sorted: audio-bearing first).""" + v_with = os.path.join(tmp_dir, 'with_audio.mp4') + v_without = os.path.join(tmp_dir, 'no_audio.mp4') + make_test_video(v_with, duration=2, with_audio=True) + make_test_video(v_without, duration=2, with_audio=False) + + grid = GridCompositor(ffmpeg) + out = os.path.join(tmp_dir, 'grid.mp4') + # Put audio-bearing first (as the processor does) + grid.composite([(v_with, 0), (v_without, 0)], 2.0, out, 640, 360) + + assert prober.has_audio(out) + vol = _get_volume(out) + assert vol is not None + assert vol > -50 + + +def test_audio_merge_preserves_timing(ffmpeg, prober, tmp_dir): + """Audio tracks placed with adelay should not shift.""" + # Create a 10s video + vid = os.path.join(tmp_dir, 'video.mp4') + make_test_video(vid, duration=10, with_audio=True) + + # Create audio that starts at 5s + audio = os.path.join(tmp_dir, 'delayed.m4a') + make_test_audio(audio, duration=3, frequency=880) + + merger = AudioMerger(ffmpeg, prober) + final = os.path.join(tmp_dir, 'final.mp4') + merger.merge(vid, [(audio, 5.0)], tmp_dir, final, total_duration=10) + + # Final should be 10s + dur = prober.get_duration(final) + assert 9.5 < dur < 10.5 + + # Audio at 0-4s should be original only + vol_start = _get_volume_at(final, 0, 3) + # Audio at 5-8s should be louder (original + delayed) + vol_mid = _get_volume_at(final, 5, 3) + # Both should have audio + assert vol_start is not None + assert vol_mid is not None + + +def _get_volume_at(path, start, duration): + """Get mean volume at a specific time range.""" + result = subprocess.run( + ['ffmpeg', '-ss', str(start), '-i', path, + '-t', str(duration), '-af', 'volumedetect', '-f', 'null', '-'], + capture_output=True, text=True, + ) + for line in result.stderr.splitlines(): + if 'mean_volume' in line: + return float(line.split('mean_volume:')[1].strip().split()[0]) + return None + + +def test_segment_duration_matches_plan(ffmpeg, prober, tmp_dir): + """Normalized segments should match planned duration within tolerance.""" + segments = SegmentBuilder(ffmpeg, prober) + vid = os.path.join(tmp_dir, 'input.mp4') + out = os.path.join(tmp_dir, 'norm.mp4') + make_test_video(vid, duration=10) + + # Normalize with max_duration=5 + segments.normalize(vid, out, 640, 360, 'yuv420p', max_duration=5) + dur = prober.get_duration(out) + assert 4.5 < dur < 5.5 + + +def test_black_segment_has_audio(ffmpeg, prober, tmp_dir): + """Black gap segments must have a silent audio stream for concat.""" + segments = SegmentBuilder(ffmpeg, prober) + path = os.path.join(tmp_dir, 'black.mp4') + segments.generate_black(path, 3.0, 640, 360) + assert prober.has_audio(path) + vol = _get_volume(path) + assert vol is not None + assert vol < -80 # should be silent From 644e7827870f58d0369b143c51eadf0dad007dff Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 10 Apr 2026 22:37:32 +0000 Subject: [PATCH 53/65] Extract audio from non-primary grid webcams, add audio pipeline tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grid input 0 already has audio — other webcams need extraction. Tracks which path is input 0 per grid segment and excludes only those. Added test_audio_pipeline.py: echo, timing, duration, silence tests. --- ..._audio_pipeline.cpython-311-pytest-9.0.3.pyc | Bin 0 -> 17493 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__pycache__/test_audio_pipeline.cpython-311-pytest-9.0.3.pyc diff --git a/tests/__pycache__/test_audio_pipeline.cpython-311-pytest-9.0.3.pyc b/tests/__pycache__/test_audio_pipeline.cpython-311-pytest-9.0.3.pyc new file mode 100644 index 0000000000000000000000000000000000000000..df12f38c7ba73832b3caff141c292b838bac5693 GIT binary patch literal 17493 zcmeHPTW}OtdhVI&x!*NHAc@Nefz4nf3lJFAEXL+ySZrdj*G^=wXNT#Q7?_KEx(AH5 zXIF8yY88qt&u+QOla!OplLIjiJo=G5CQma|Q&x2qRjwqi6mKMKUUs z_RM(cudgB&y&j?Bp9y$aNpL1eOF}at`1O^;mB>t_5}k?CaI;8j;vBD%hjU1BdeAw3{z5+x~xpz!`V2M^BT!Y;ewpUCi53dl3cShX!ALb5j%FQ zGB3}asaLBqfqe8;St=D?Bek-&c(&c9mxpAI5B%pWMUwAUO4>azP`a)X&b7T2WRM!$;~>BcP~o z+X)oRmrE)>s*2&CE!PUF;ioM4QAue&=QG6lf_lyf5Lv00pGPVmmc!)yWh0uOli|UpvqQ$Aof!Nn*MTewU&Ys6eRnyW zZceubTlIGJru6x-TZcQLl!@Kq2-8WyQm+>M}D z(1KdPHNub$q){zy&S&Bgp4o)+UORKvGYijc6rNk_0fXO1*xlE=_PwM!)@wjZ z!p^(l#>T$g8Q+ZNp}}6~Y9sZSW7y=XpVm_C0jqucuNHt2Wntg-XkINX`EHE4_g%-T zO*`kQF+f-Cz$33(K=wUlo1bvi4miiO>Z*0&VAXE^-mKd3Mg~@3W&z$ojX`Zd8?>$5 zJC0HIc0#stw;=laG^gLgT{z=>LZq#q_}IQHJtaM@h0R@K%_Hm@Yn&}PbGBV0(-@Re zpdf5pcmY+hY|+%k@?1e-T|^q&u)^+~l8ZEq0B9PZ zh8nb6)K{*HhHw7z6p~#?b|cwy>bmD)0Im)${kmsi2pfP#vy4N#h4#!wKwY@*CA(nc zWE#lBgmb)P4>0dxF|_G92+a*x%^l26m_u_^+6zs?AoebVL5(7X$;x!$-aJer^s)__ zF~YM%eys%xI76XcL)HQ&=ULd$8v>mjoVy8n zUDGg6IUl>6-2ldsOfT54={YvM=W-dtTT{p;1c~Qsr7FQ&O;Ch2!u%mJ!u(Dcu|xBh z^JhV;S4&mJ2$X9@czTQoaH&C|K#X;H_RQspJWuj-<=R=ej%uMI8{xcCn3a)jBwnNs z1lTr?5>g{ZCJ&ljrJ&~J_vg!nDz$o0?afH%p@%YU3;Am%`G6Yyqe^)rjW^86Hool=4(Uwr9qIJyE;%rswWr?06Uak?u`>*6#d z?wZVcN8HmD_vqptNIUEJEG6pJj*(q7JE5F3Mopjj7+XE_G?wHjv)*%%zui8_V=HCPd= z0cq9-ZwWa3yqg?p?dvAD=*cZGJENOA;-;m2t?5n_Q|!DWZqltZ<4}aSscRK51kVD9 zahCRRULS*lT5wx?9YC7(!71ZFebxl<8QxxtgO}G1cf{c)X%^vUXrrK;GibqAl4>O`w68C%g4d+_F8s4q~VFqtoi8ak9YiW zN9)qHXm?;*ADCV|iBJE@YhyQ+j`)06d|nryhs0O$b*-WHP&d9!k8fK%b~l!|`tpab zTzRFnw*5>uHmS!Z7mqHBk>9=Y!7I&mt+%g@ch*mKMxVX8_VcZs*G@0K(iP9>;+c+k z=Bw0b>u5K%Sx;?VJh8k!+Yz%%`)CUDwB!9~$Ac9j{bUeUy=yz(z zqZ;}8g=17BH}!B8T(73$O?`H=C8DU{!~>UGcAHWosx1A4l~0_N3#3$?19u)ka+3>* zp2|(t^B4&IRNtT~I(m2BAWs85Mz#Z`s<@?49wIxCHGyOj3DuN*pgNPCkUd5AB6A-Q z!-pE?PcZvD5>$N20VF>~@&b}WNL~bzOOcb9M)i@rf`n_2f`8kqtByR%AC3~72`Ey_xfs1*(ZX?Woama5Yx{TxrNE|Ln8Jd$4`DIhru zL_s5#OPOMd6m_iUZHg4$lsujBJiRGo9;)K?E8F4P(1Vz~`qqc*IQGX_>#cURo7$tN_AH)2>G4?m^i9yCK&;)Ti~Asf%oEo6JWKFZ!jg07#k;TKyLn9q=WNCL( zT6zHz?7Sn6=vJC>5CEyJRlpED3q_2x1lp-tw~xU=O^Jh8n?UU+lrk(I2gBQIas2YY zU`HHmZf<7a2ZousvL6!cydw_kR+@1T0I9B3zz{qOMT~>jVb<+qa8PqI2eCGR+D|BD zSUwJhx7V`cdAeWpMVk_C4-fwP+Yx=y{$n2C(Di~VLF{vKSOEBa1V@VB<-`yj(IFy6 zq1*`d^}%09;AHG~q&%(@JM0<*I4hLQUT(<$|Q`bni5JP>4q%_zFYj9`4 zy`9oh;NG^+M(^IO)l%TzUWdp&r_S&b-Vv9}bJaWI!oeLGc8s=n?_7?5X~cAIr=?M8 zy_PO9_x8Xe+!2>^ZpAy2f|Y=?n0(|}1Z3YVZg|33OghK5>MXi&FpC?$KeGs~sf`Oe zY0zMnTE~?vnE8vclttk#Tae^(;j-OVrJO6xs*?+08lXr8Tzm?cfV&W-aBx;zCX4!V zz!7uZhmMq;{5n>V)zaR1OD=Z8Xm1iISv$GV$o`CCd;YgXHH*Xp;(A*VPK5*S{!Uoz~rYlL9Aur zTDqHhMo&Gnc;c(X`qt0eFSg(7CUSZrxA^jxKX~K!^2o-HxN+&Z)|O5bQ|!DWZq%(b z<4}aSv1=7D1kVD9ah9Irygmj8wYG2&@F30l;FNKoK5K$uU_ka-9Ly5^VtWGC92MKq zkNq($8pf3jk{vs z;S?2VptmT+G?4~++r@NoWszEPiD^=z7mB#DDDCWVyT)C&-Y;Ah_|q0^$~x0Wh=ixGfY@W&1enO=DbXnyCT-hhB>3=-7> zriK}kq9A7nw6K|n;84&wBcO3|9m*b94T~DWO+n*~V;dlIyERUGEw1ZmF3k@@lW`rG zN6)P6T(C53+uk16Ox=1kNal82uU}W_F-YdlaO@t^t=}X~jWa5xAP_lf>NhbJ7o~YP z`otAa4=v`Lac2%R&H>OggQnIQ0X=5`Hl57E;ZqodQ!XulV6Lzacoa)2hv1yzIh5Mr ziy$d$iJ(+|hi_2ST3Lqes3?QXP7SBjL>;O^dzc~whL!WMyUId0X2 z1UG)#nN*C$D$EwzXJcgyyP=eEP-=D9Wo-=JS5x62)`Pug&VdG5n~Vd7VtXxD6??XS z=MTb-L9ygP3f_B8@#wqQ`C(sPQS<>@AXq>p(P??3AF*(;tl@``Sp1sk+zT23EdU>} zz&Ca(x23^ei`PYu6?VfFElF;XZ&f>m>th$#4Fjv#DO_(2DfomVbY1zXBXr?lguzut z=-Sz(kogfy&~Ec_)a|`JTyG=iU4b1MZGrwqz`ob_-8&GM;k_B6_9?&ScUk%P+W@|H z^n;ZzipW0uRO|_7)n&(Bbyi(CnAMnLW>&ZI#Z4<;L`r~-IbyyAzr8sQ!$$>SEes#A z1Ym|NndIx{Gfn4n0MS3-`HV^#Y0xHB*zE6)JJWk+ze%c~&n!H*VUX~`AW^JYIQSBn z`7UJV3iCaIqEZK!_qoDFaD=n3l|i$iUp0f$Ar~t2Gv{KhS}dRe@GRD+BE&PWAFbra zX71;mP-^pIGheXNk9`lF&>)1M;Sb`cc?kCb9>PryK@oFpuP93xi8&d*hc2jco{sDP z;NW%(6yz#)xd{oim_33wdlgw?J(VTsxbl`n!Q-~cAD{;+iTo>w`u$P_7}K8n@W7P= ztzdh!8=cUj6N^Wd#n>0(=xuSdb)-EGp}XMKozlgrjySb!`kBW}Kl2e53)~TbF@}_$ zcf>JWgxETyb`Hkqnmh)vF#yMSG-%g>;*~fC;};ze8-s&jL>vW7NV6id8*6}e3;@`P zerBl~8Py}BKl+)eH`(s^?Q<*lF_`Zd9N6EjpE!nj1>PQeUg1= zPf*UXv)M-~1l42s4;HKs*e98d9mOjn(c{uj6wr!b4D^^>)1X?-IF!=(ej1WO?cvP0 zg8Bhn-q!^xpHj`@Vdy}mCe_RGLGt%d4w{3YWPyS`#p78P-s}i(F5^D1%D?a;{kJTn zJJz`@Jl|RATo!U2=eaCw>p0J4;pvX^T&dPXM{u0?17T02kNza=5 h0w-J6c{be_IN2M)%AAZuJfoO3$w&z6_7ggT{|n449_s)A literal 0 HcmV?d00001 From 9a8a657059c24430adad890997d02b84271ed08c Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Fri, 10 Apr 2026 23:15:55 +0000 Subject: [PATCH 54/65] Rewrite audio pipeline to use mediasession events like JS player MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace all guesswork (overlap detection, dedup, speaker switching) with StreamTimeline that builds playback windows from API mediasession events — matching exactly what the MTS-Link web player does. - Add StreamTimeline class with dataclasses (MediaSession, TimeWindow, GridSource, AudioTrack, DownloadChunk, SlideEvent) - GridCompositor now mixes all audio streams inline via amix - Remove dedup strategy, overlap heuristics, webcam audio extraction - Remove dead code: deduplicate, extract_admin_conf_ids, is_valid, is_silent, analyze_audio_levels, legacy compat wrappers - Fix .gitignore (was too broad, ignored tests/) - 49 tests passing --- .gitignore | 4 +- mtslinker/audio.py | 33 ++- mtslinker/grid.py | 61 +++++- mtslinker/prober.py | 70 ------- mtslinker/processor.py | 441 ++++++++++++----------------------------- mtslinker/segments.py | 122 +----------- mtslinker/timeline.py | 406 +++++++++++++++++++++++++++++++++++++ mtslinker/webinar.py | 4 +- tests/test_segments.py | 4 - tests/test_timeline.py | 187 +++++++++++++++++ 10 files changed, 797 insertions(+), 535 deletions(-) create mode 100644 mtslinker/timeline.py create mode 100644 tests/test_timeline.py diff --git a/.gitignore b/.gitignore index 232bef8..eab9fc4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ -*/ *.log *.mp4 *.mp3 -!mtslinker !get_sessionId.mp4 -!setup.py__pycache__/ +__pycache__/ diff --git a/mtslinker/audio.py b/mtslinker/audio.py index a1e0c03..0f2c18f 100644 --- a/mtslinker/audio.py +++ b/mtslinker/audio.py @@ -2,10 +2,11 @@ import os import shutil import subprocess -from typing import List, Tuple +from typing import List, Union from mtslinker.ffmpeg import FFmpegRunner from mtslinker.prober import MediaProber +from mtslinker.timeline import AudioTrack class AudioMerger: @@ -17,25 +18,35 @@ def __init__(self, ffmpeg: FFmpegRunner, prober: MediaProber): self.ffmpeg = ffmpeg self.prober = prober - def merge(self, video_path: str, audio_files: List[Tuple[str, float]], - tmp_dir: str, output_path: str, total_duration: float = 0) -> str: + def merge(self, video_path: str, + audio_files: List[Union[AudioTrack, tuple]], + tmp_dir: str, output_path: str, + total_duration: float = 0) -> str: video_duration = total_duration or self.prober.get_duration(video_path) + # Normalize to AudioTrack objects + tracks = [] + for af in audio_files: + if isinstance(af, AudioTrack): + tracks.append(af) + else: + tracks.append(AudioTrack(path=af[0], start_time=af[1])) + # Step 1: Mix in batches with inline adelay batch_outputs = [] - total_batches = (len(audio_files) + self.BATCH_SIZE - 1) // self.BATCH_SIZE + total_batches = (len(tracks) + self.BATCH_SIZE - 1) // self.BATCH_SIZE for batch_idx, batch_start in enumerate( - range(0, len(audio_files), self.BATCH_SIZE) + range(0, len(tracks), self.BATCH_SIZE) ): - batch = audio_files[batch_start:batch_start + self.BATCH_SIZE] + batch = tracks[batch_start:batch_start + self.BATCH_SIZE] batch_out = os.path.join(tmp_dir, f'audio_batch_{batch_idx}.m4a') inputs = [] filter_parts = [] mix_labels = [] - for j, (apath, start_time) in enumerate(batch): - inputs.extend(['-i', apath]) - delay_ms = int(start_time * 1000) + for j, track in enumerate(batch): + inputs.extend(['-i', track.path]) + delay_ms = int(track.start_time * 1000) label = f'a{j}' filter_parts.append( f'[{j}:a]adelay={delay_ms}|{delay_ms},' @@ -65,8 +76,8 @@ def merge(self, video_path: str, audio_files: List[Tuple[str, float]], batch_out, ], description=f'amix batch {batch_idx+1} ' - f'({len(batch)} tracks, offset {batch[0][1]:.0f}-' - f'{batch[-1][1]:.0f}s)', + f'({len(batch)} tracks, offset {batch[0].start_time:.0f}-' + f'{batch[-1].start_time:.0f}s)', ) batch_outputs.append(batch_out) except subprocess.CalledProcessError: diff --git a/mtslinker/grid.py b/mtslinker/grid.py index 9815332..073c173 100644 --- a/mtslinker/grid.py +++ b/mtslinker/grid.py @@ -1,7 +1,8 @@ import math -import os +from typing import List, Union from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.timeline import GridSource class GridCompositor: @@ -18,9 +19,29 @@ def compute_layout(self, n: int): def even(self, x: int) -> int: return x & ~1 - def composite(self, active_segments: list, duration: float, - output_path: str, target_w: int, target_h: int) -> str: - n = len(active_segments) + def composite(self, active_segments: List[Union[GridSource, tuple]], + duration: float, output_path: str, + target_w: int, target_h: int, + mix_all_audio: bool = False) -> str: + """Composite multiple webcam streams into a grid. + + Args: + active_segments: List of GridSource objects or legacy (path, offset) + tuples. + mix_all_audio: When True, amix audio from all inputs. When False, + only map audio from input 0 (legacy behavior). + """ + # Normalize to GridSource objects + sources = [] + for seg in active_segments: + if isinstance(seg, GridSource): + sources.append(seg) + elif len(seg) == 3: + sources.append(GridSource(path=seg[0], offset=seg[1], has_audio=seg[2])) + else: + sources.append(GridSource(path=seg[0], offset=seg[1])) + + n = len(sources) cols, rows = self.compute_layout(n) cell_w = self.even(target_w // cols) cell_h = self.even(target_h // rows) @@ -29,8 +50,8 @@ def composite(self, active_segments: list, duration: float, filter_parts = [] labels = [] - for i, (path, offset) in enumerate(active_segments): - inputs.extend(['-ss', str(offset), '-i', path]) + for i, src in enumerate(sources): + inputs.extend(['-ss', str(src.offset), '-i', src.path]) label = f'v{i}' filter_parts.append( f'[{i}:v]scale=w=min({cell_w}\\,iw):h=min({cell_h}\\,ih)' @@ -41,7 +62,7 @@ def composite(self, active_segments: list, duration: float, total_cells = cols * rows for i in range(n, total_cells): - idx = len(active_segments) + i - n + idx = len(sources) + i - n inputs.extend(['-f', 'lavfi', '-i', f'color=black:s={cell_w}x{cell_h}:d={duration}:r=25']) labels.append(f'[{idx}:v]') @@ -61,12 +82,36 @@ def composite(self, active_segments: list, duration: float, + f'xstack=inputs={total_cells}:layout={layout}[out]' ) + # Audio mixing + if mix_all_audio and n > 1: + audio_labels = [] + for i, src in enumerate(sources): + alabel = f'a{i}' + if src.has_audio: + filter_graph += ( + f';[{i}:a]apad=whole_dur={duration},' + f'atrim=0:{duration}[{alabel}]' + ) + else: + filter_graph += ( + f';anullsrc=r=44100:cl=stereo[null{i}]' + f';[null{i}]atrim=0:{duration}[{alabel}]' + ) + audio_labels.append(f'[{alabel}]') + filter_graph += ( + ';' + ''.join(audio_labels) + + f'amix=inputs={n}:duration=longest:normalize=0[aout]' + ) + audio_map = ['-map', '[aout]'] + else: + audio_map = ['-map', '0:a?'] + cmd = [ 'ffmpeg', '-y', '-v', 'error', *inputs, '-t', str(duration), '-filter_complex', filter_graph, - '-map', '[out]', '-map', '0:a?', + '-map', '[out]', *audio_map, *self.ffmpeg.get_video_encoder_fast(), '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', diff --git a/mtslinker/prober.py b/mtslinker/prober.py index da81746..6c95ef2 100644 --- a/mtslinker/prober.py +++ b/mtslinker/prober.py @@ -1,8 +1,5 @@ import json -import logging -import os import subprocess -import tempfile from concurrent.futures import ThreadPoolExecutor, as_completed from typing import List, Tuple @@ -20,17 +17,6 @@ def probe_streams(self, file_path: str) -> dict: return {} return json.loads(result.stdout) - def is_valid(self, file_path: str) -> bool: - result = subprocess.run( - ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', - '-of', 'csv=p=0', file_path], - capture_output=True, text=True, - ) - if result.returncode != 0: - logging.warning(f'Corrupt/invalid file: {file_path}: {result.stderr.strip()[:200]}') - return False - return True - def has_video(self, file_path: str) -> bool: info = self.probe_streams(file_path) return any(s.get('codec_type') == 'video' for s in info.get('streams', [])) @@ -59,62 +45,6 @@ def get_video_params(self, file_path: str) -> Tuple[int, int, str]: return w, h, pix_fmt return 1920, 1080, 'yuv420p' - def is_silent(self, file_path: str, threshold: float = -88.0) -> bool: - result = subprocess.run( - ['ffmpeg', '-v', 'error', '-i', file_path, - '-t', '10', '-af', 'volumedetect', '-f', 'null', '-'], - capture_output=True, text=True, - ) - for line in result.stderr.splitlines(): - if 'mean_volume' in line: - try: - vol = float(line.split('mean_volume:')[1].strip().split()[0]) - return vol < threshold - except (ValueError, IndexError): - pass - return True - - def analyze_audio_levels(self, file_path: str, window_sec: float = 2.0, - sample_rate: int = 44100) -> List[Tuple[float, float]]: - reset_samples = int(sample_rate * window_sec) - with tempfile.NamedTemporaryFile(suffix='.txt', delete=False) as tmp: - tmp_path = tmp.name - try: - result = subprocess.run( - ['ffmpeg', '-v', 'error', '-i', file_path, - '-af', f'astats=metadata=1:reset={reset_samples},' - f'ametadata=print:key=lavfi.astats.Overall.RMS_level' - f':file={tmp_path}', - '-f', 'null', '-'], - capture_output=True, text=True, - ) - if result.returncode != 0: - return [] - levels = [] - current_time = None - with open(tmp_path) as f: - for line in f: - line = line.strip() - if line.startswith('frame:'): - for part in line.split(): - if part.startswith('pts_time:'): - try: - current_time = float(part.split(':')[1]) - except (ValueError, IndexError): - pass - elif 'RMS_level' in line and current_time is not None: - try: - val = float(line.split('=')[1]) - levels.append((current_time, val)) - except (ValueError, IndexError): - pass - return levels - finally: - try: - os.remove(tmp_path) - except OSError: - pass - def probe_file(self, file_path: str) -> dict: info = self.probe_streams(file_path) streams = info.get('streams', []) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index de259ab..9994112 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -13,9 +13,7 @@ import os import shutil import subprocess -from collections import defaultdict -from concurrent.futures import ThreadPoolExecutor, as_completed -from typing import Dict, List, Tuple, Union +from typing import List from mtslinker.ffmpeg import FFmpegRunner from mtslinker.prober import MediaProber @@ -23,6 +21,7 @@ from mtslinker.grid import GridCompositor from mtslinker.slides import SlideCompositor from mtslinker.audio import AudioMerger +from mtslinker.timeline import StreamTimeline, GridSource, AudioTrack class VideoProcessor: @@ -37,8 +36,13 @@ def __init__(self): self.audio = AudioMerger(self.ffmpeg, self.prober) def analyze(self, total_duration, downloaded_files, directory, - output_path, max_duration=None, slide_events=None): - """Phase 1: Probe all files, make all decisions, write manifest.""" + output_path, max_duration=None, slide_events=None, + timeline=None): + """Phase 1: Probe all files, make all decisions, write manifest. + + Uses the StreamTimeline built from mediasession events as the source + of truth — matches what the JS player does. + """ logging.info('Phase 1: Analyzing files...') all_files = self.prober.probe_all_files(downloaded_files) @@ -79,211 +83,83 @@ def analyze(self, total_duration, downloaded_files, directory, target_h = MAX_H logging.info(f'Target resolution: {target_w}x{target_h}, pix_fmt={target_pix_fmt}') - # Detect overlaps - has_overlaps = False - for i in range(1, len(video_files)): - prev_dur = video_files[i-1].get('api_duration', 0) or video_files[i-1]['duration'] - prev_end = video_files[i-1]['start_time'] + prev_dur - if video_files[i]['start_time'] < prev_end - 0.5: - has_overlaps = True - break - - # Decide strategy and build segment plan - overlap_strategy = 'none' - kept_video = [] - extra_audio = [] + # Build segment plan from timeline (mediasession events). + # Each stream gets an independent playback element, all active + # streams play simultaneously — no dedup/overlap guessing needed. segments = [] - - if has_overlaps and slide_events: - overlap_strategy = 'dedup' - vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), - f.get('is_admin', False)) for f in video_files] - deduped = self.segments.deduplicate(vf_tuples) - kept_paths = {v[0] for v in deduped} - - dedup_map = {f['path']: f for f in video_files} - for path, start_time in deduped: - if path in dedup_map: - kept_video.append(dedup_map[path]) - - for f in video_files: - if f['path'] not in kept_paths and f['has_audio']: - extra_audio.append(f) - - logging.info(f'Dedup: kept {len(kept_video)}, ' - f'{len(extra_audio)} extra audio from webcams') - - elif has_overlaps: - overlap_strategy = 'grid' - annotated = [] - for f in video_files: - dur = f.get('api_duration', 0) or f['duration'] - annotated.append((f['path'], f['start_time'], dur, - f['start_time'] + dur, f['has_audio'])) - - event_times_set = set() - for _, start, _, end, _ in annotated: - event_times_set.add(start) - event_times_set.add(end) - event_times_set.add(total_duration) - event_times = sorted(event_times_set) - - grid_windows = [] - prev_active_key = None - window_start = None - window_active = None - for ei in range(len(event_times) - 1): - t_start = event_times[ei] - t_end = event_times[ei + 1] - if t_end - t_start < 0.1: - continue - active = [] - for path, seg_start, dur, seg_end, has_audio in annotated: - if seg_start < t_end and seg_end > t_start: - offset = max(0, t_start - seg_start) - remaining = dur - offset - if remaining > 0.5: - active.append({'path': path, 'offset': offset, - 'remaining': remaining, - 'has_audio': has_audio}) - active_key = tuple(a['path'] for a in active) - if active_key == prev_active_key and window_start is not None: + timeline.map_downloaded_files(downloaded_files) + tw_windows = timeline.get_windows_with_files() + path_lookup = {f['path']: f for f in video_files} + + for tw in tw_windows: + sources = [] + for stream in tw.streams: + if not stream.file_path or stream.file_path not in path_lookup: continue - if prev_active_key is not None and window_start is not None: - grid_windows.append({ - 'start_time': window_start, - 'duration': t_start - window_start, - 'sources': window_active, - }) - window_start = t_start - prev_active_key = active_key - window_active = active - if window_start is not None and window_active: - grid_windows.append({ - 'start_time': window_start, - 'duration': event_times[-1] - window_start, - 'sources': window_active, + offset = max(0, tw.start_time - stream.start_time) + f = path_lookup[stream.file_path] + sources.append({ + 'path': stream.file_path, + 'offset': offset, + 'remaining': tw.duration, + 'has_audio': stream.has_audio and f['has_audio'], }) - - for gw in grid_windows: - if not gw['sources']: - segments.append({ - 'type': 'gap', - 'duration': gw['duration'], - 'start_time': gw['start_time'], - }) - else: - segments.append({ - 'type': 'grid', - 'start_time': gw['start_time'], - 'planned_duration': gw['duration'], - 'sources': gw['sources'], - }) - - # Grid composite includes audio from input 0 (sorted by has_audio). - # Extract audio from OTHER webcams (not input 0) to avoid echo - # but still capture all voices. - grid_input0_paths = set() - for seg in segments: - if seg['type'] == 'grid' and seg.get('sources'): - # Input 0 = first source sorted by has_audio (same as execute) - sorted_sources = sorted(seg['sources'], - key=lambda s: not s.get('has_audio', False)) - if sorted_sources: - grid_input0_paths.add(sorted_sources[0]['path']) - - for f in video_files: - if f['has_audio'] and f['path'] not in grid_input0_paths: - extra_audio.append(f) - if extra_audio: - logging.info(f'Grid: {len(extra_audio)} non-primary webcam audio to extract') - - else: - vf_tuples = [(f['path'], f['start_time'], f.get('conf_id'), - f.get('is_admin', False)) for f in video_files] - deduped = self.segments.deduplicate(vf_tuples) - dedup_map = {f['path']: f for f in video_files} - for path, start_time in deduped: - if path in dedup_map: - kept_video.append(dedup_map[path]) - - # Build segment plan for non-grid strategies - if overlap_strategy != 'grid': - segments = [] - current_time = 0.0 - for i, f in enumerate(kept_video): - start_time = f['start_time'] - if start_time < current_time - 0.5: - errors.append(f'Segment {i} overlaps at {start_time:.1f}s (current={current_time:.1f}s)') - continue - - gap = start_time - current_time - if gap > 0.1: + if not sources: segments.append({ - 'type': 'gap', 'duration': gap, 'start_time': current_time, + 'type': 'gap', + 'duration': tw.duration, + 'start_time': tw.start_time, + }) + elif len(sources) == 1: + s = sources[0] + f = path_lookup[s['path']] + segments.append({ + 'type': 'video', + 'source_path': s['path'], + 'start_time': tw.start_time, + 'source_offset': s['offset'], + 'source_duration': f['duration'], + 'planned_duration': tw.duration, + 'has_audio': s['has_audio'], + 'width': f['width'], + 'height': f['height'], + }) + else: + segments.append({ + 'type': 'grid', + 'start_time': tw.start_time, + 'planned_duration': tw.duration, + 'sources': sources, + 'mix_all_audio': True, }) - max_dur = 0 - for j in range(i + 1, len(kept_video)): - ns = kept_video[j]['start_time'] - if ns > start_time + 0.5: - max_dur = ns - start_time - break - - file_dur = f.get('api_duration', 0) or f['duration'] - planned_dur = max_dur if max_dur > 0 else file_dur - segments.append({ - 'type': 'video', - 'source_path': f['path'], - 'start_time': start_time, - 'source_duration': f['duration'], - 'max_duration': max_dur, - 'planned_duration': planned_dur, - 'has_audio': f['has_audio'], - 'width': f['width'], - 'height': f['height'], - }) - current_time = start_time + planned_dur - - # Fill grid timeline gaps - if overlap_strategy == 'grid' and segments: - filled = [] - current_time = 0.0 - for s in segments: - start = s.get('start_time', 0) - if start - current_time > 0.1: - filled.append({ - 'type': 'gap', - 'duration': start - current_time, - 'start_time': current_time, - }) - filled.append(s) - current_time = start + s.get('planned_duration', s.get('duration', 0)) - segments = filled - - # Trailing gap - if overlap_strategy == 'grid': - current_time = 0 - for s in segments: - end = s.get('start_time', 0) + s.get('planned_duration', s.get('duration', 0)) - if end > current_time: - current_time = end + # Fill leading/trailing gaps + filled = [] + current_time = 0.0 + for s in segments: + start = s.get('start_time', 0) + if start - current_time > 0.1: + filled.append({ + 'type': 'gap', + 'duration': start - current_time, + 'start_time': current_time, + }) + filled.append(s) + current_time = start + s.get('planned_duration', s.get('duration', 0)) if current_time < total_duration - 0.1: - segments.append({ + filled.append({ 'type': 'gap', 'duration': total_duration - current_time, 'start_time': current_time, }) + segments = filled + + logging.info(f'Timeline: {len(segments)} segments from mediasession events') - # Audio tracks + # Audio tracks — only audio-only files (webcam audio is mixed + # inline by GridCompositor, no separate extraction needed) all_audio = [{'path': f['path'], 'start_time': f['start_time'], 'origin': 'original'} for f in audio_files] - for f in extra_audio: - all_audio.append({ - 'path': f['path'], - 'start_time': f['start_time'], - 'origin': 'webcam_extract', - }) # Slide compositing plan slide_plan = None @@ -323,7 +199,7 @@ def analyze(self, total_duration, downloaded_files, directory, 'output_path': output_path, 'max_duration': max_duration, 'target': {'width': target_w, 'height': target_h, 'pix_fmt': target_pix_fmt}, - 'overlap_strategy': overlap_strategy, + 'overlap_strategy': 'timeline', 'segments': segments, 'audio_tracks': all_audio, 'slide_compositing': slide_plan, @@ -334,8 +210,8 @@ def analyze(self, total_duration, downloaded_files, directory, 'total_files': len(all_files), 'video_files': len(video_files), 'audio_files': len(audio_files), - 'kept_video': len(kept_video), - 'extra_audio': len(extra_audio), + 'kept_video': 0, + 'extra_audio': 0, 'skipped': skipped, 'segments': len(segments), 'gaps': sum(1 for s in segments if s['type'] == 'gap'), @@ -374,31 +250,11 @@ def execute(self, manifest): tmp_dir = os.path.join(directory, '_tmp_ffmpeg') os.makedirs(tmp_dir, exist_ok=True) - # Extract audio from dropped webcam files - audio_files = [] - extra_dir = os.path.join(tmp_dir, 'extra_audio') - os.makedirs(extra_dir, exist_ok=True) - extract_count = 0 - for track in manifest['audio_tracks']: - if track['origin'] == 'webcam_extract': - audio_path = os.path.join(extra_dir, f'extra_{extract_count}.m4a') - try: - self.ffmpeg.run( - ['ffmpeg', '-y', '-v', 'error', - '-i', track['path'], '-vn', - '-c:a', 'aac', '-b:a', '128k', - '-ar', '44100', '-ac', '2', - audio_path], - description=f'extract audio from webcam {extract_count}', - ) - audio_files.append((audio_path, track['start_time'])) - extract_count += 1 - except subprocess.CalledProcessError: - logging.warning(f'Failed to extract audio from {track["path"]}') - else: - audio_files.append((track['path'], track['start_time'])) - if extract_count: - logging.info(f'Extracted audio from {extract_count} webcam files') + # Collect audio-only tracks (webcam audio is mixed inline by grid) + audio_files = [ + AudioTrack(path=t['path'], start_time=t['start_time'], origin=t['origin']) + for t in manifest['audio_tracks'] + ] # Process segments segments = manifest['segments'] @@ -419,13 +275,18 @@ def execute(self, manifest): duration = seg['planned_duration'] sources = sorted(seg['sources'], key=lambda s: not s.get('has_audio', False)) - active = [(s['path'], s['offset']) for s in sources] + mix_audio = seg.get('mix_all_audio', False) + active = [ + GridSource(path=s['path'], offset=s['offset'], + has_audio=s.get('has_audio', True)) + for s in sources + ] try: if len(active) == 1: - path, offset = active[0] + src = active[0] self.ffmpeg.run( ['ffmpeg', '-y', '-v', 'error', - '-ss', str(offset), '-i', path, + '-ss', str(src.offset), '-i', src.path, '-t', str(duration), '-vf', f'scale={target_w}:{target_h}:' f'force_original_aspect_ratio=decrease,' @@ -437,7 +298,9 @@ def execute(self, manifest): description=f'grid {i} (1 webcam, {duration:.0f}s)', ) else: - self.grid.composite(active, duration, seg_path, target_w, target_h) + self.grid.composite(active, duration, seg_path, + target_w, target_h, + mix_all_audio=mix_audio) with_audio = os.path.join(tmp_dir, f'grida_{i}.mp4') final_seg = self.segments.ensure_audio(seg_path, with_audio) concat_segments.append(final_seg) @@ -450,14 +313,18 @@ def execute(self, manifest): elif seg['type'] == 'video': norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') - self.segments.normalize(seg['source_path'], norm_path, - target_w, target_h, target_pix_fmt, - max_duration=seg.get('max_duration', 0)) + source_offset = seg.get('source_offset', 0) + planned_dur = seg.get('planned_duration', 0) + self.segments.normalize( + seg['source_path'], norm_path, + target_w, target_h, target_pix_fmt, + max_duration=planned_dur, + seek=source_offset, + ) with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') final_seg = self.segments.ensure_audio(norm_path, with_audio_path) actual_dur = self.prober.get_duration(final_seg) - planned_dur = seg.get('planned_duration', 0) if planned_dur > 0 and actual_dur < planned_dur - 1.0: shortfall = planned_dur - actual_dur logging.warning( @@ -518,11 +385,12 @@ def execute(self, manifest): return result_path def process(self, total_duration, downloaded_files, directory, - output_path, max_duration=None, slide_events=None): + output_path, max_duration=None, slide_events=None, + timeline=None): """Full pipeline: analyze then execute.""" manifest = self.analyze( total_duration, downloaded_files, directory, output_path, - max_duration, slide_events, + max_duration, slide_events, timeline=timeline, ) return self.execute(manifest) @@ -530,101 +398,38 @@ def process(self, total_duration, downloaded_files, directory, # --- Backward-compatible module-level functions --- def process_and_download_clips(directory, json_data): - """Parse API JSON for chunks, slides, admin IDs.""" - segments_builder = SegmentBuilder(FFmpegRunner.__new__(FFmpegRunner), MediaProber()) + """Parse API JSON for chunks, slides, admin IDs. + + Returns: + (total_duration, tagged_chunks, deduped_slides, timeline) + where timeline is a StreamTimeline instance built from mediasession events. + tagged_chunks are DownloadChunk-compatible tuples for backward compatibility + with the download pipeline. + """ total_duration = float(json_data.get('duration', 0)) if not total_duration: raise ValueError('Duration not found in JSON data.') - admin_conf_ids = segments_builder.extract_admin_conf_ids(json_data) - - mediasession_adds = {} - chunks = [] - slide_events = [] - for event in json_data.get('eventLogs', []): - if not isinstance(event, dict): - continue - module = event.get('module', '') - data = event.get('data', {}) - if not isinstance(data, dict): - continue - - if module == 'mediasession.add' and 'url' in data: - ms_id = data.get('id') - stream = data.get('stream', {}) - conf_id = None - if isinstance(stream, dict): - conf = stream.get('conference', {}) - if isinstance(conf, dict): - conf_id = conf.get('id') - mediasession_adds[ms_id] = { - 'url': data['url'], - 'start': event.get('relativeTime', 0), - 'conf_id': conf_id, - 'api_duration': 0, - } + timeline = StreamTimeline() + timeline.build(json_data) - if module == 'mediasession.update': - ms_id = data.get('id') - if ms_id in mediasession_adds: - mediasession_adds[ms_id]['api_duration'] = data.get('duration', 0) - - elif 'url' in data and module not in ('mediasession.add',): - stream = data.get('stream', {}) - conf_id = None - if isinstance(stream, dict): - conf = stream.get('conference', {}) - if isinstance(conf, dict): - conf_id = conf.get('id') - url = data['url'] - if not any(ms['url'] == url for ms in mediasession_adds.values()): - chunks.append((url, event.get('relativeTime', 0), conf_id, 0)) - - if module == 'presentation.update': - fr = data.get('fileReference', {}) - if not isinstance(fr, dict): - continue - slide = fr.get('slide', {}) - if not isinstance(slide, dict) or not slide.get('url'): - continue - slide_events.append({ - 'time': event.get('relativeTime', 0), - 'slide_number': slide.get('number', 0), - 'slide_url': slide['url'], - }) + # Convert to tuples for backward compatibility with download_chunks_parallel + tagged_chunks = [ + (c.url, c.start_time, c.conf_id, c.is_admin, c.api_duration) + for c in timeline.get_download_chunks() + ] - tagged_chunks = [] - for ms in mediasession_adds.values(): - tagged_chunks.append(( - ms['url'], ms['start'], ms['conf_id'], - ms['conf_id'] in admin_conf_ids, - ms['api_duration'], - )) - for url, start, conf_id, api_dur in chunks: - tagged_chunks.append((url, start, conf_id, conf_id in admin_conf_ids, api_dur)) + deduped_slides = timeline.get_slide_events_as_dicts() - deduped_slides = [] - for se in slide_events: - if not deduped_slides or se['slide_url'] != deduped_slides[-1]['slide_url']: - deduped_slides.append(se) - - if deduped_slides: - logging.info(f'Found {len(deduped_slides)} presentation slide changes') - - return total_duration, tagged_chunks, deduped_slides + return total_duration, tagged_chunks, deduped_slides, timeline def compile_final_video(total_duration, downloaded_files, directory, - output_path, max_duration, slide_events=None): + output_path, max_duration, slide_events=None, + timeline=None): """Backward-compatible entry point.""" processor = VideoProcessor() processor.process(total_duration, downloaded_files, directory, - output_path, max_duration, slide_events) - - -# Keep old names for backward compatibility with webinar.py -def analyze_video(*args, **kwargs): - return VideoProcessor().analyze(*args, **kwargs) + output_path, max_duration, slide_events, + timeline=timeline) -def execute_video(manifest): - return VideoProcessor().execute(manifest) diff --git a/mtslinker/segments.py b/mtslinker/segments.py index 911884f..cfccca2 100644 --- a/mtslinker/segments.py +++ b/mtslinker/segments.py @@ -1,7 +1,4 @@ -import logging import os -from collections import defaultdict -from typing import Dict, List, Tuple from mtslinker.ffmpeg import FFmpegRunner from mtslinker.prober import MediaProber @@ -55,11 +52,13 @@ def ensure_audio(self, input_path: str, output_path: str) -> str: def normalize(self, input_path: str, output_path: str, width: int, height: int, pix_fmt: str, - max_duration: float = 0) -> str: + max_duration: float = 0, seek: float = 0) -> str: + seek_args = ['-ss', str(seek)] if seek > 0 else [] duration_args = ['-t', str(max_duration)] if max_duration > 0 else [] self.ffmpeg.run( [ 'ffmpeg', '-y', '-v', 'error', + *seek_args, '-i', input_path, '-vf', f'scale={width}:{height}:force_original_aspect_ratio=decrease,' f'pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', @@ -74,119 +73,4 @@ def normalize(self, input_path: str, output_path: str, ) return output_path - def deduplicate(self, video_files: list) -> List[Tuple[str, float]]: - if not video_files: - return video_files - annotated = [] - for item in video_files: - path, start = item[0], item[1] - conf_id = item[2] if len(item) > 2 else None - is_admin = item[3] if len(item) > 3 else False - dur = self.prober.get_duration(path) - annotated.append((path, start, dur, start + dur, conf_id, is_admin)) - - conf_total_dur = defaultdict(float) - conf_is_admin = {} - for _, _, dur, _, conf_id, is_admin in annotated: - if conf_id: - conf_total_dur[conf_id] += dur - if is_admin: - conf_is_admin[conf_id] = True - - admin_confs = {c for c in conf_total_dur if conf_is_admin.get(c)} - if admin_confs: - main_conf = max(admin_confs, key=conf_total_dur.get) - logging.info( - f'Dedup: main conference {main_conf} (ADMIN, ' - f'{conf_total_dur[main_conf]:.0f}s total from ' - f'{sum(1 for x in annotated if x[4] == main_conf)} segments)' - ) - elif conf_total_dur: - main_conf = max(conf_total_dur, key=conf_total_dur.get) - logging.info( - f'Dedup: main conference {main_conf} ' - f'({conf_total_dur[main_conf]:.0f}s total from ' - f'{sum(1 for x in annotated if x[4] == main_conf)} segments)' - ) - else: - main_conf = None - - main_segs = sorted( - [s for s in annotated if s[4] == main_conf], key=lambda x: x[1]) - other_segs = sorted( - [s for s in annotated if s[4] != main_conf], key=lambda x: x[1]) - - kept = [] - for seg in main_segs: - if not kept or seg[1] >= kept[-1][3] - 0.5: - kept.append(seg) - else: - if seg[2] > kept[-1][2]: - kept[-1] = seg - - filled = [] - for i, seg in enumerate(kept): - gap_start = kept[i - 1][3] if i > 0 else 0 - gap_end = seg[1] - if gap_end - gap_start > 1.0: - best = None - for other in other_segs: - if other[3] <= gap_start or other[1] >= gap_end: - continue - overlap_start = max(other[1], gap_start) - overlap_end = min(other[3], gap_end) - overlap_dur = overlap_end - overlap_start - if overlap_dur <= 0: - continue - if best is None or overlap_dur > best[2]: - best = (other[0], overlap_start, overlap_dur, - overlap_end, other[4], other[5]) - if best: - filled.append(best) - filled.append(seg) - - result = [(path, start) for path, start, dur, end, conf_id, is_admin in filled] - logging.info(f'Dedup: {len(annotated)} -> {len(result)} segments ' - f'(removed {len(annotated) - len(result)} overlapping)') - return result - - def extract_admin_conf_ids(self, json_data: Dict) -> set: - admin_user_ids = set() - user_to_conf = {} - - for event in json_data.get('eventLogs', []): - if not isinstance(event, dict): - continue - module = event.get('module', '') - data_list = event.get('data', []) - if isinstance(data_list, dict): - data_list = [data_list] - if not isinstance(data_list, list): - continue - - for d in data_list: - if not isinstance(d, dict): - continue - if 'userlist' in module: - role = d.get('role', '') - user = d.get('user', {}) - if isinstance(user, dict) and role == 'ADMIN': - uid = user.get('id') - if uid: - admin_user_ids.add(uid) - if module == 'conference.add': - user = d.get('user', {}) - if isinstance(user, dict): - uid = user.get('id') - cid = d.get('id') - if uid and cid: - user_to_conf.setdefault(uid, set()).add(cid) - - admin_confs = set() - for uid in admin_user_ids: - admin_confs.update(user_to_conf.get(uid, set())) - - if admin_confs: - logging.info(f'Found {len(admin_confs)} conference(s) from ADMIN users') - return admin_confs diff --git a/mtslinker/timeline.py b/mtslinker/timeline.py new file mode 100644 index 0000000..a66e6d2 --- /dev/null +++ b/mtslinker/timeline.py @@ -0,0 +1,406 @@ +"""StreamTimeline — builds a playback timeline from API mediasession events. + +Replicates what the MTS-Link web player does: uses mediasession.add/update +events to know exactly which files to play and when, and conference events +to know which streams have audio/video. +""" +import logging +from dataclasses import dataclass, field +from typing import Dict, List, Optional + + +@dataclass +class MediaSession: + """A single media session (one recorded file).""" + id: str + url: str + start_time: float + duration: float = 0.0 + conf_id: Optional[str] = None + has_audio: bool = True + has_video: bool = True + user_id: Optional[str] = None + file_path: Optional[str] = None # set after download + + +@dataclass +class SlideEvent: + """A presentation slide change event.""" + time: float + slide_number: int + slide_url: str + local_path: Optional[str] = None # set after download + + +@dataclass +class DownloadChunk: + """A media chunk to download.""" + url: str + start_time: float + conf_id: Optional[str] = None + is_admin: bool = False + api_duration: float = 0.0 + session_id: Optional[str] = None + + +@dataclass +class GridSource: + """A source stream for grid compositing.""" + path: str + offset: float + has_audio: bool = True + + +@dataclass +class AudioTrack: + """An audio track to merge into the final video.""" + path: str + start_time: float + origin: str = 'original' # 'original' or 'webcam_extract' + + +@dataclass +class TimeWindow: + """A time window where the set of active streams is constant.""" + start_time: float + duration: float + streams: List[MediaSession] = field(default_factory=list) + + @property + def end_time(self) -> float: + return self.start_time + self.duration + + @property + def has_video(self) -> bool: + return any(s.has_video for s in self.streams) + + @property + def has_audio(self) -> bool: + return any(s.has_audio for s in self.streams) + + @property + def stream_count(self) -> int: + return len(self.streams) + + +class StreamTimeline: + """Builds a timeline of active streams from API event logs. + + Mirrors the web player's approach: each mediasession gets an independent + playback element, and all active streams play simultaneously. + """ + + def __init__(self): + self.sessions: Dict[str, MediaSession] = {} + self.admin_conf_ids: set = set() + self.total_duration: float = 0.0 + self.slide_events: List[SlideEvent] = [] + + def build(self, json_data: dict) -> List[TimeWindow]: + """Parse API JSON and build timeline windows. + + Args: + json_data: Full API response from /api/eventsessions/{sid}/record + + Returns: + List of TimeWindow objects in chronological order. + """ + self.total_duration = float(json_data.get('duration', 0)) + self._extract_admin_info(json_data) + self._extract_sessions(json_data) + self._extract_slides(json_data) + windows = self._compute_windows() + return windows + + def _extract_admin_info(self, json_data: dict): + """Extract admin user IDs and their conference IDs.""" + admin_user_ids = set() + user_to_conf = {} + + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data_list = event.get('data', []) + if isinstance(data_list, dict): + data_list = [data_list] + if not isinstance(data_list, list): + continue + + for d in data_list: + if not isinstance(d, dict): + continue + if 'userlist' in module: + role = d.get('role', '') + user = d.get('user', {}) + if isinstance(user, dict) and role == 'ADMIN': + uid = user.get('id') + if uid: + admin_user_ids.add(uid) + if module == 'conference.add': + user = d.get('user', {}) + if isinstance(user, dict): + uid = user.get('id') + cid = d.get('id') + if uid and cid: + user_to_conf.setdefault(uid, set()).add(cid) + + for uid in admin_user_ids: + self.admin_conf_ids.update(user_to_conf.get(uid, set())) + + def _extract_sessions(self, json_data: dict): + """Extract mediasession.add/update events into MediaSession objects.""" + conf_users = {} + conf_props = {} + + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data = event.get('data', {}) + if not isinstance(data, dict): + continue + + if module == 'conference.add': + cid = data.get('id') + user = data.get('user', {}) + if isinstance(user, dict) and cid: + uid = user.get('id') + if uid: + conf_users[cid] = uid + conf_props[cid] = { + 'has_audio': data.get('hasAudio', True), + 'has_video': data.get('hasVideo', True), + } + + elif module == 'conference.update': + cid = data.get('id') + if cid and cid in conf_props: + if 'hasAudio' in data: + conf_props[cid]['has_audio'] = data['hasAudio'] + if 'hasVideo' in data: + conf_props[cid]['has_video'] = data['hasVideo'] + + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data = event.get('data', {}) + if not isinstance(data, dict): + continue + + if module == 'mediasession.add' and 'url' in data: + ms_id = data.get('id') + if not ms_id: + continue + conf_id = None + stream = data.get('stream', {}) + if isinstance(stream, dict): + conf = stream.get('conference', {}) + if isinstance(conf, dict): + conf_id = conf.get('id') + + props = conf_props.get(conf_id, {}) + self.sessions[ms_id] = MediaSession( + id=ms_id, + url=data['url'], + start_time=event.get('relativeTime', 0), + conf_id=conf_id, + has_audio=props.get('has_audio', True), + has_video=props.get('has_video', True), + user_id=conf_users.get(conf_id), + ) + + elif module == 'mediasession.update': + ms_id = data.get('id') + if ms_id and ms_id in self.sessions: + if 'duration' in data: + self.sessions[ms_id].duration = float(data['duration']) + + # Fallback: pick up URLs not covered by mediasession events + known_urls = {s.url for s in self.sessions.values()} + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data = event.get('data', {}) + if not isinstance(data, dict): + continue + if module == 'mediasession.add': + continue + url = data.get('url') + if url and url not in known_urls: + fallback_id = f'fallback_{len(self.sessions)}' + conf_id = None + stream = data.get('stream', {}) + if isinstance(stream, dict): + conf = stream.get('conference', {}) + if isinstance(conf, dict): + conf_id = conf.get('id') + self.sessions[fallback_id] = MediaSession( + id=fallback_id, + url=url, + start_time=event.get('relativeTime', 0), + conf_id=conf_id, + ) + known_urls.add(url) + + logging.info(f'Timeline: {len(self.sessions)} media sessions extracted') + + def _extract_slides(self, json_data: dict): + """Extract presentation.update events into SlideEvent objects.""" + raw_slides = [] + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + if event.get('module') != 'presentation.update': + continue + data = event.get('data', {}) + if not isinstance(data, dict): + continue + fr = data.get('fileReference', {}) + if not isinstance(fr, dict): + continue + slide = fr.get('slide', {}) + if not isinstance(slide, dict) or not slide.get('url'): + continue + raw_slides.append(SlideEvent( + time=event.get('relativeTime', 0), + slide_number=slide.get('number', 0), + slide_url=slide['url'], + )) + + # Deduplicate consecutive identical slides + self.slide_events = [] + for se in raw_slides: + if not self.slide_events or se.slide_url != self.slide_events[-1].slide_url: + self.slide_events.append(se) + + if self.slide_events: + logging.info(f'Timeline: {len(self.slide_events)} presentation slide changes') + + def _compute_windows(self) -> List[TimeWindow]: + """Compute time windows where the set of active streams is constant.""" + if not self.sessions: + return [] + + boundaries = set() + boundaries.add(0.0) + if self.total_duration > 0: + boundaries.add(self.total_duration) + + for s in self.sessions.values(): + boundaries.add(s.start_time) + end = s.start_time + s.duration if s.duration > 0 else self.total_duration + boundaries.add(end) + + sorted_times = sorted(boundaries) + + windows = [] + prev_streams_key = None + merge_start = None + merge_streams = None + + for i in range(len(sorted_times) - 1): + t_start = sorted_times[i] + t_end = sorted_times[i + 1] + if t_end - t_start < 0.1: + continue + + active = [] + for s in self.sessions.values(): + s_end = s.start_time + s.duration if s.duration > 0 else self.total_duration + if s.start_time < t_end and s_end > t_start: + active.append(s) + + streams_key = tuple(s.id for s in sorted(active, key=lambda x: x.id)) + + if streams_key == prev_streams_key and merge_start is not None: + continue + else: + if merge_start is not None: + windows.append(TimeWindow( + start_time=merge_start, + duration=t_start - merge_start, + streams=merge_streams, + )) + merge_start = t_start + merge_streams = active + prev_streams_key = streams_key + + if merge_start is not None and sorted_times: + windows.append(TimeWindow( + start_time=merge_start, + duration=sorted_times[-1] - merge_start, + streams=merge_streams, + )) + + windows = [w for w in windows if w.duration > 0.1] + + logging.info(f'Timeline: {len(windows)} time windows computed') + return windows + + def get_download_chunks(self) -> List[DownloadChunk]: + """Get list of chunks to download as DownloadChunk objects.""" + result = [] + for s in self.sessions.values(): + result.append(DownloadChunk( + url=s.url, + start_time=s.start_time, + conf_id=s.conf_id, + is_admin=s.conf_id in self.admin_conf_ids, + api_duration=s.duration, + session_id=s.id, + )) + return sorted(result, key=lambda x: x.start_time) + + def get_slide_events_as_dicts(self) -> List[dict]: + """Get slide events as dicts (compatible with existing download pipeline).""" + return [ + {'time': se.time, 'slide_number': se.slide_number, + 'slide_url': se.slide_url} + for se in self.slide_events + ] + + def map_downloaded_files(self, downloaded_files: list): + """Map downloaded files back to sessions. + + Args: + downloaded_files: List of DownloadedFile-like objects or + (path, start_time, conf_id, is_admin) tuples. + """ + time_conf_to_session = {} + for s in self.sessions.values(): + key = (round(s.start_time, 1), s.conf_id) + time_conf_to_session[key] = s + + for item in downloaded_files: + if hasattr(item, 'path'): + path, start_time, conf_id = item.path, item.start_time, item.conf_id + else: + path = item[0] + start_time = item[1] + conf_id = item[2] if len(item) > 2 else None + + key = (round(start_time, 1), conf_id) + session = time_conf_to_session.get(key) + if session: + session.file_path = path + else: + for s in self.sessions.values(): + if abs(s.start_time - start_time) < 0.5 and s.file_path is None: + s.file_path = path + break + + def get_windows_with_files(self) -> List[TimeWindow]: + """Get windows filtered to only include streams with downloaded files.""" + result = [] + for w in self._compute_windows(): + filed_streams = [s for s in w.streams if s.file_path] + if filed_streams: + result.append(TimeWindow( + start_time=w.start_time, + duration=w.duration, + streams=filed_streams, + )) + return result diff --git a/mtslinker/webinar.py b/mtslinker/webinar.py index ee4ecda..fffdc69 100644 --- a/mtslinker/webinar.py +++ b/mtslinker/webinar.py @@ -24,7 +24,7 @@ def fetch_webinar_data(event_sessions: str, record_id: str, session_id=None, max directory = create_directory_if_not_exists(sanitized_name) output_video_path = os.path.join(directory, f'{sanitized_name}.mp4') - total_duration, chunks, slide_events = process_and_download_clips(directory, json_data) + total_duration, chunks, slide_events, timeline = process_and_download_clips(directory, json_data) logging.info(f'Found {len(chunks)} chunks to download ({total_duration} sec total)') # Download all chunks in parallel @@ -38,7 +38,7 @@ def fetch_webinar_data(event_sessions: str, record_id: str, session_id=None, max compile_final_video( total_duration, downloaded_files, directory, output_video_path, - max_duration, slide_events=downloaded_slides, + max_duration, slide_events=downloaded_slides, timeline=timeline, ) logging.info(f'Final video saved to {output_video_path}') diff --git a/tests/test_segments.py b/tests/test_segments.py index 01c3b77..fc2c881 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -42,7 +42,3 @@ def test_normalize_with_max_duration(segments, tmp_dir): from mtslinker.prober import MediaProber dur = MediaProber().get_duration(out) assert dur < 3.0 - - -def test_deduplicate_empty(segments): - assert segments.deduplicate([]) == [] diff --git a/tests/test_timeline.py b/tests/test_timeline.py new file mode 100644 index 0000000..2f34c1e --- /dev/null +++ b/tests/test_timeline.py @@ -0,0 +1,187 @@ +from mtslinker.timeline import StreamTimeline, MediaSession, TimeWindow, DownloadChunk + + +def _make_json(events, duration=100.0): + return {'duration': duration, 'eventLogs': events} + + +def _ms_add(ms_id, url, time, conf_id=None): + data = {'id': ms_id, 'url': url} + if conf_id: + data['stream'] = {'conference': {'id': conf_id}} + return {'module': 'mediasession.add', 'relativeTime': time, 'data': data} + + +def _ms_update(ms_id, duration): + return {'module': 'mediasession.update', 'relativeTime': 0, + 'data': {'id': ms_id, 'duration': duration}} + + +def _conf_add(conf_id, user_id, has_audio=True, has_video=True): + return {'module': 'conference.add', 'relativeTime': 0, + 'data': {'id': conf_id, 'user': {'id': user_id}, + 'hasAudio': has_audio, 'hasVideo': has_video}} + + +def test_single_session(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 100.0), + ]) + tl = StreamTimeline() + windows = tl.build(json_data) + assert len(windows) == 1 + assert windows[0].start_time == 0.0 + assert abs(windows[0].duration - 100.0) < 0.2 + assert len(windows[0].streams) == 1 + assert windows[0].streams[0].url == 'http://a.mp4' + + +def test_two_overlapping_sessions(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 60.0), + _ms_add('ms2', 'http://b.mp4', 10.0), + _ms_update('ms2', 50.0), + ], duration=60.0) + tl = StreamTimeline() + windows = tl.build(json_data) + # Windows: [0-10] ms1 only, [10-60] ms1+ms2 + assert len(windows) == 2 + assert len(windows[0].streams) == 1 + assert len(windows[1].streams) == 2 + + +def test_sequential_sessions(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 30.0), + _ms_add('ms2', 'http://b.mp4', 30.0), + _ms_update('ms2', 30.0), + ], duration=60.0) + tl = StreamTimeline() + windows = tl.build(json_data) + assert len(windows) == 2 + assert len(windows[0].streams) == 1 + assert windows[0].streams[0].url == 'http://a.mp4' + assert len(windows[1].streams) == 1 + assert windows[1].streams[0].url == 'http://b.mp4' + + +def test_gap_between_sessions(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 20.0), + _ms_add('ms2', 'http://b.mp4', 40.0), + _ms_update('ms2', 20.0), + ], duration=60.0) + tl = StreamTimeline() + windows = tl.build(json_data) + # [0-20] ms1, [20-40] empty, [40-60] ms2 + assert len(windows) == 3 + assert len(windows[0].streams) == 1 + assert len(windows[1].streams) == 0 + assert len(windows[2].streams) == 1 + + +def test_conference_audio_video_flags(): + json_data = _make_json([ + _conf_add('c1', 'u1', has_audio=True, has_video=False), + _ms_add('ms1', 'http://a.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + ], duration=50.0) + tl = StreamTimeline() + windows = tl.build(json_data) + assert len(windows) == 1 + s = windows[0].streams[0] + assert s.has_audio is True + assert s.has_video is False + + +def test_admin_detection(): + json_data = _make_json([ + {'module': 'userlist.add', 'relativeTime': 0, + 'data': {'role': 'ADMIN', 'user': {'id': 'u1'}}}, + _conf_add('c1', 'u1'), + _ms_add('ms1', 'http://a.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + assert 'c1' in tl.admin_conf_ids + + +def test_get_download_chunks(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + _ms_add('ms2', 'http://b.mp4', 10.0, conf_id='c2'), + _ms_update('ms2', 40.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + dl = tl.get_download_chunks() + assert len(dl) == 2 + assert dl[0].url == 'http://a.mp4' + assert dl[0].start_time == 0.0 + assert dl[1].url == 'http://b.mp4' + + +def test_map_downloaded_files(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + tl.map_downloaded_files([('/tmp/a.mp4', 0.0, 'c1', False, 50.0)]) + assert tl.sessions['ms1'].file_path == '/tmp/a.mp4' + + +def test_three_webcam_overlap(): + """Three webcams with staggered starts — should produce multiple windows.""" + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 100.0), + _ms_add('ms2', 'http://b.mp4', 20.0), + _ms_update('ms2', 80.0), + _ms_add('ms3', 'http://c.mp4', 40.0), + _ms_update('ms3', 60.0), + ], duration=100.0) + tl = StreamTimeline() + windows = tl.build(json_data) + # [0-20] 1 stream, [20-40] 2 streams, [40-100] 3 streams + assert len(windows) == 3 + assert len(windows[0].streams) == 1 + assert len(windows[1].streams) == 2 + assert len(windows[2].streams) == 3 + + +def test_empty_events(): + json_data = _make_json([]) + tl = StreamTimeline() + windows = tl.build(json_data) + assert windows == [] + + +def test_fallback_url_capture(): + """URLs not in mediasession.add should be captured as fallback sessions.""" + json_data = _make_json([ + {'module': 'someother.event', 'relativeTime': 5.0, + 'data': {'url': 'http://fallback.mp4'}}, + ], duration=50.0) + tl = StreamTimeline() + windows = tl.build(json_data) + assert len(tl.sessions) == 1 + assert any('fallback' in sid for sid in tl.sessions) + + +def test_time_window_properties(): + w = TimeWindow(start_time=10.0, duration=5.0, streams=[ + MediaSession(id='1', url='a', start_time=0, has_video=True, has_audio=False), + MediaSession(id='2', url='b', start_time=0, has_video=False, has_audio=True), + ]) + assert w.end_time == 15.0 + assert w.has_video is True + assert w.has_audio is True + assert w.stream_count == 2 From dca9856209f0225edbda03be2598631b1c48dd31 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sun, 12 Apr 2026 17:51:01 +0000 Subject: [PATCH 55/65] Fix silent audio: always download via HLS, not raw storage URL Audio-only streams were downloaded as raw binary from the storage URL, which returns valid MP4 containers with silent audio (-91 dB). The real audio lives in the HLS playlist variants. Now tries HLS first for all streams (video and audio-only), falling back to direct download only if HLS is unavailable. --- mtslinker/downloader.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/mtslinker/downloader.py b/mtslinker/downloader.py index 7819830..c2f178e 100644 --- a/mtslinker/downloader.py +++ b/mtslinker/downloader.py @@ -22,18 +22,18 @@ def _storage_to_hls_url(storage_url: str) -> str: ) -def _hls_has_video(hls_url: str) -> bool: - """Check if an HLS playlist has a video track.""" +def _hls_is_available(hls_url: str) -> bool: + """Check if an HLS playlist is reachable and non-empty.""" try: with httpx.Client(timeout=httpx.Timeout(10)) as client: r = client.get(hls_url, headers={'User-Agent': 'Mozilla/5.0'}) - return 'v1/' in r.text + return r.status_code == 200 and '#EXTM3U' in r.text except Exception: return False def download_hls_chunk(hls_url: str, save_path: str) -> str: - """Download an HLS stream to mp4 using ffmpeg.""" + """Download an HLS stream (video, audio, or both) to mp4 using ffmpeg.""" if not os.path.exists(save_path): subprocess.run( [ @@ -106,12 +106,19 @@ def download_video_chunk(video_url: str, save_directory: str, max_retries: int = os.remove(file_path) for attempt in range(max_retries + 1): - # Check if HLS version has video (storage mp4 may be audio-only) hls_url = _storage_to_hls_url(video_url) - if _hls_has_video(hls_url): - logging.info(f'HLS has video for {filename}, downloading via ffmpeg') - download_hls_chunk(hls_url, file_path) - else: + hls_ok = _hls_is_available(hls_url) + if hls_ok: + logging.info(f'Downloading via HLS: {filename}') + try: + download_hls_chunk(hls_url, file_path) + except subprocess.CalledProcessError: + logging.warning(f'HLS download failed for {filename}, falling back to direct') + if os.path.exists(file_path): + os.remove(file_path) + hls_ok = False + if not hls_ok: + logging.info(f'Downloading direct: {filename}') with open(file_path, 'wb') as file: with httpx.Client(timeout=TIMEOUT_SETTINGS) as client: with client.stream('GET', video_url) as response: From b8cc77f50cdcf94e02a9c80c5edbfe23c93081d1 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sun, 12 Apr 2026 18:32:31 +0000 Subject: [PATCH 56/65] Fix NameError: remove stale overlap_strategy reference in log line The variable was removed in the mediasession rewrite (9a8a657) but the logging line still referenced it. Strategy is now always 'timeline'. --- mtslinker/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 9994112..a4f2e3d 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -225,7 +225,7 @@ def analyze(self, total_duration, downloaded_files, directory, logging.info(f'Plan: {manifest["stats"]["segments"]} segments ' f'({manifest["stats"]["gaps"]} gaps), ' f'{len(all_audio)} audio tracks, ' - f'strategy={overlap_strategy}') + f'strategy=timeline') if errors: for e in errors: From 12f48d01d6942fdc0f491b824a891cb443f900f5 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sun, 12 Apr 2026 22:57:42 +0000 Subject: [PATCH 57/65] Add presenter layout: admin full-screen, participant PIP top-right MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When multiple streams are active: - Screenshare → main area, admin → PIP overlay - Admin (no screenshare) → main area, participant → PIP overlay - No admin/screenshare → fall back to grid Also fix grid xstack to always output exact target resolution, preventing concat corruption from mismatched segment sizes. --- mtslinker/grid.py | 97 +++++++++++++++++++++++++++++++++++++++++- mtslinker/processor.py | 82 ++++++++++++++++++++++++++++++++--- mtslinker/timeline.py | 1 + 3 files changed, 171 insertions(+), 9 deletions(-) diff --git a/mtslinker/grid.py b/mtslinker/grid.py index 073c173..96f0389 100644 --- a/mtslinker/grid.py +++ b/mtslinker/grid.py @@ -1,5 +1,5 @@ import math -from typing import List, Union +from typing import List, Optional, Union from mtslinker.ffmpeg import FFmpegRunner from mtslinker.timeline import GridSource @@ -79,7 +79,9 @@ def composite(self, active_segments: List[Union[GridSource, tuple]], filter_graph += ';' filter_graph += ( ''.join(labels) - + f'xstack=inputs={total_cells}:layout={layout}[out]' + + f'xstack=inputs={total_cells}:layout={layout}[stacked]' + + f';[stacked]scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[out]' ) # Audio mixing @@ -120,3 +122,94 @@ def composite(self, active_segments: List[Union[GridSource, tuple]], ] self.ffmpeg.run(cmd, description=f'grid {n} webcams, {duration:.0f}s') return output_path + + def presenter_composite(self, main: GridSource, + overlay: Optional[GridSource], + extra_audio: List[GridSource], + duration: float, output_path: str, + target_w: int, target_h: int, + mix_all_audio: bool = True) -> str: + """Composite presenter layout: main full-screen, overlay PIP top-right. + + Output is always exactly target_w x target_h. + """ + pip_w = self.even(target_w // 4) + pip_h = self.even(target_h // 4) + margin = 16 + + inputs = ['-ss', str(main.offset), '-i', main.path] + all_sources = [main] + + if overlay: + inputs.extend(['-ss', str(overlay.offset), '-i', overlay.path]) + all_sources.append(overlay) + + for src in extra_audio: + inputs.extend(['-ss', str(src.offset), '-i', src.path]) + all_sources.append(src) + + # Video filter + vfilter = ( + f'[0:v]scale={target_w}:{target_h}:' + f'force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[main]' + ) + + if overlay: + vfilter += ( + f';[1:v]scale={pip_w}:{pip_h}:' + f'force_original_aspect_ratio=decrease,' + f'pad={pip_w}:{pip_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[pip]' + f';[main][pip]overlay=x={target_w - pip_w - margin}:y={margin}[out]' + ) + else: + vfilter += ';[main]copy[out]' + + # Audio filter — mix all sources that have audio + audio_labels = [] + afilter = '' + for i, src in enumerate(all_sources): + if not src.has_audio: + continue + alabel = f'a{i}' + afilter += ( + f';[{i}:a]apad=whole_dur={duration},' + f'atrim=0:{duration}[{alabel}]' + ) + audio_labels.append(f'[{alabel}]') + + if len(audio_labels) > 1: + afilter += ( + ';' + ''.join(audio_labels) + + f'amix=inputs={len(audio_labels)}:duration=longest:normalize=0[aout]' + ) + audio_map = ['-map', '[aout]'] + elif len(audio_labels) == 1: + # Single audio, keep the label and map it + afilter += f';{audio_labels[0]}acopy[aout]' + audio_map = ['-map', '[aout]'] + else: + afilter = '' + audio_map = [] + + filter_graph = vfilter + afilter + + cmd = [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-t', str(duration), + '-filter_complex', filter_graph, + '-map', '[out]', *audio_map, + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + output_path, + ] + desc = 'presenter' + if overlay: + desc += ' + PIP' + if extra_audio: + desc += f' + {len(extra_audio)} audio' + self.ffmpeg.run(cmd, description=f'{desc}, {duration:.0f}s') + return output_path diff --git a/mtslinker/processor.py b/mtslinker/processor.py index a4f2e3d..6826438 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -98,11 +98,14 @@ def analyze(self, total_duration, downloaded_files, directory, continue offset = max(0, tw.start_time - stream.start_time) f = path_lookup[stream.file_path] + is_admin = stream.conf_id in timeline.admin_conf_ids sources.append({ 'path': stream.file_path, 'offset': offset, 'remaining': tw.duration, 'has_audio': stream.has_audio and f['has_audio'], + 'is_admin': is_admin, + 'is_screenshare': bool(stream.has_video and not stream.has_audio), }) if not sources: segments.append({ @@ -125,13 +128,43 @@ def analyze(self, total_duration, downloaded_files, directory, 'height': f['height'], }) else: - segments.append({ - 'type': 'grid', - 'start_time': tw.start_time, - 'planned_duration': tw.duration, - 'sources': sources, - 'mix_all_audio': True, - }) + # Presenter layout: admin/screenshare as main, others as PIP + screenshares = [s for s in sources if s.get('is_screenshare')] + admins = [s for s in sources if s.get('is_admin') and not s.get('is_screenshare')] + others = [s for s in sources if not s.get('is_admin') and not s.get('is_screenshare')] + + main_source = None + overlay_source = None + if screenshares: + main_source = screenshares[0] + overlay_source = admins[0] if admins else (others[0] if others else None) + elif admins: + main_source = admins[0] + overlay_source = others[0] if others else None + + if main_source: + used = {id(main_source)} + if overlay_source: + used.add(id(overlay_source)) + extra_audio = [s for s in sources if id(s) not in used and s.get('has_audio')] + segments.append({ + 'type': 'presenter', + 'start_time': tw.start_time, + 'planned_duration': tw.duration, + 'main_source': main_source, + 'overlay_source': overlay_source, + 'extra_audio_sources': extra_audio, + 'mix_all_audio': True, + }) + else: + # No admin, no screenshare — fall back to grid + segments.append({ + 'type': 'grid', + 'start_time': tw.start_time, + 'planned_duration': tw.duration, + 'sources': sources, + 'mix_all_audio': True, + }) # Fill leading/trailing gaps filled = [] @@ -311,6 +344,41 @@ def execute(self, manifest): target_w, target_h, target_pix_fmt) concat_segments.append(gap_path) + elif seg['type'] == 'presenter': + os.makedirs(grid_dir, exist_ok=True) + seg_path = os.path.join(grid_dir, f'presenter_{i}.mp4') + duration = seg['planned_duration'] + ms = seg['main_source'] + main = GridSource(path=ms['path'], offset=ms['offset'], + has_audio=ms.get('has_audio', True), + is_admin=ms.get('is_admin', False)) + overlay = None + if seg.get('overlay_source'): + os_ = seg['overlay_source'] + overlay = GridSource(path=os_['path'], offset=os_['offset'], + has_audio=os_.get('has_audio', True), + is_admin=os_.get('is_admin', False)) + extra = [ + GridSource(path=s['path'], offset=s['offset'], + has_audio=s.get('has_audio', True)) + for s in seg.get('extra_audio_sources', []) + ] + try: + self.grid.presenter_composite( + main, overlay, extra, duration, seg_path, + target_w, target_h, + mix_all_audio=seg.get('mix_all_audio', True), + ) + with_audio = os.path.join(tmp_dir, f'presenta_{i}.mp4') + final_seg = self.segments.ensure_audio(seg_path, with_audio) + concat_segments.append(final_seg) + except subprocess.CalledProcessError: + logging.warning(f'Presenter segment {i} failed, using black gap') + gap_path = os.path.join(tmp_dir, f'presfail_{i}.mp4') + self.segments.generate_black(gap_path, duration, + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + elif seg['type'] == 'video': norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') source_offset = seg.get('source_offset', 0) diff --git a/mtslinker/timeline.py b/mtslinker/timeline.py index a66e6d2..c863ad2 100644 --- a/mtslinker/timeline.py +++ b/mtslinker/timeline.py @@ -49,6 +49,7 @@ class GridSource: path: str offset: float has_audio: bool = True + is_admin: bool = False @dataclass From 2b74d277f1d36860486de4aff5d8b8435a7b332a Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Sun, 12 Apr 2026 23:02:19 +0000 Subject: [PATCH 58/65] Refactor grid.py into GridLayout, PresenterLayout, and facade Split GridCompositor into smaller classes: - GridLayout: xstack grid compositing - PresenterLayout: main + PIP overlay compositing - GridCompositor: backward-compatible facade delegating to both - _build_audio_filter: shared audio mixing helper - _even: shared utility Add 12 new tests covering presenter layout (main-only, PIP, extra audio, resolution consistency), audio filter builder, grid resolution matching, and facade backward compat. 61 tests passing. --- mtslinker/grid.py | 166 ++++++++++++++++++++++++---------------- tests/test_grid.py | 184 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 267 insertions(+), 83 deletions(-) diff --git a/mtslinker/grid.py b/mtslinker/grid.py index 96f0389..21fdc3f 100644 --- a/mtslinker/grid.py +++ b/mtslinker/grid.py @@ -5,46 +5,63 @@ from mtslinker.timeline import GridSource -class GridCompositor: - """Composites multiple webcam streams into a grid layout.""" +def _even(x: int) -> int: + """Round down to nearest even number.""" + return x & ~1 + + +def _build_audio_filter(sources: List[GridSource], duration: float): + """Build amix filter graph and map args for a list of sources. + + Returns (filter_str, audio_map) where filter_str starts with ';' + and audio_map is e.g. ['-map', '[aout]']. + """ + audio_labels = [] + afilter = '' + for i, src in enumerate(sources): + if not src.has_audio: + continue + alabel = f'a{i}' + afilter += ( + f';[{i}:a]apad=whole_dur={duration},' + f'atrim=0:{duration}[{alabel}]' + ) + audio_labels.append(f'[{alabel}]') + + if len(audio_labels) > 1: + afilter += ( + ';' + ''.join(audio_labels) + + f'amix=inputs={len(audio_labels)}:duration=longest:normalize=0[aout]' + ) + return afilter, ['-map', '[aout]'] + elif len(audio_labels) == 1: + afilter += f';{audio_labels[0]}acopy[aout]' + return afilter, ['-map', '[aout]'] + else: + return '', [] + + +class GridLayout: + """Composites multiple streams into an equal-size grid using xstack.""" def __init__(self, ffmpeg: FFmpegRunner): self.ffmpeg = ffmpeg - def compute_layout(self, n: int): + @staticmethod + def compute_layout(n: int): cols = math.ceil(math.sqrt(n)) rows = math.ceil(n / cols) return cols, rows - def even(self, x: int) -> int: - return x & ~1 - - def composite(self, active_segments: List[Union[GridSource, tuple]], + def composite(self, sources: List[GridSource], duration: float, output_path: str, target_w: int, target_h: int, mix_all_audio: bool = False) -> str: - """Composite multiple webcam streams into a grid. - - Args: - active_segments: List of GridSource objects or legacy (path, offset) - tuples. - mix_all_audio: When True, amix audio from all inputs. When False, - only map audio from input 0 (legacy behavior). - """ - # Normalize to GridSource objects - sources = [] - for seg in active_segments: - if isinstance(seg, GridSource): - sources.append(seg) - elif len(seg) == 3: - sources.append(GridSource(path=seg[0], offset=seg[1], has_audio=seg[2])) - else: - sources.append(GridSource(path=seg[0], offset=seg[1])) - + """Composite streams into a grid. Output is exactly target_w x target_h.""" n = len(sources) cols, rows = self.compute_layout(n) - cell_w = self.even(target_w // cols) - cell_h = self.even(target_h // rows) + cell_w = _even(target_w // cols) + cell_h = _even(target_h // rows) inputs = [] filter_parts = [] @@ -84,7 +101,7 @@ def composite(self, active_segments: List[Union[GridSource, tuple]], + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[out]' ) - # Audio mixing + # Audio if mix_all_audio and n > 1: audio_labels = [] for i, src in enumerate(sources): @@ -123,18 +140,21 @@ def composite(self, active_segments: List[Union[GridSource, tuple]], self.ffmpeg.run(cmd, description=f'grid {n} webcams, {duration:.0f}s') return output_path - def presenter_composite(self, main: GridSource, - overlay: Optional[GridSource], - extra_audio: List[GridSource], - duration: float, output_path: str, - target_w: int, target_h: int, - mix_all_audio: bool = True) -> str: - """Composite presenter layout: main full-screen, overlay PIP top-right. - Output is always exactly target_w x target_h. - """ - pip_w = self.even(target_w // 4) - pip_h = self.even(target_h // 4) +class PresenterLayout: + """Composites presenter layout: main full-screen, optional PIP top-right.""" + + def __init__(self, ffmpeg: FFmpegRunner): + self.ffmpeg = ffmpeg + + def composite(self, main: GridSource, + overlay: Optional[GridSource], + extra_audio: List[GridSource], + duration: float, output_path: str, + target_w: int, target_h: int) -> str: + """Composite presenter layout. Output is exactly target_w x target_h.""" + pip_w = _even(target_w // 4) + pip_h = _even(target_h // 4) margin = 16 inputs = ['-ss', str(main.offset), '-i', main.path] @@ -165,33 +185,8 @@ def presenter_composite(self, main: GridSource, else: vfilter += ';[main]copy[out]' - # Audio filter — mix all sources that have audio - audio_labels = [] - afilter = '' - for i, src in enumerate(all_sources): - if not src.has_audio: - continue - alabel = f'a{i}' - afilter += ( - f';[{i}:a]apad=whole_dur={duration},' - f'atrim=0:{duration}[{alabel}]' - ) - audio_labels.append(f'[{alabel}]') - - if len(audio_labels) > 1: - afilter += ( - ';' + ''.join(audio_labels) - + f'amix=inputs={len(audio_labels)}:duration=longest:normalize=0[aout]' - ) - audio_map = ['-map', '[aout]'] - elif len(audio_labels) == 1: - # Single audio, keep the label and map it - afilter += f';{audio_labels[0]}acopy[aout]' - audio_map = ['-map', '[aout]'] - else: - afilter = '' - audio_map = [] - + # Audio + afilter, audio_map = _build_audio_filter(all_sources, duration) filter_graph = vfilter + afilter cmd = [ @@ -213,3 +208,42 @@ def presenter_composite(self, main: GridSource, desc += f' + {len(extra_audio)} audio' self.ffmpeg.run(cmd, description=f'{desc}, {duration:.0f}s') return output_path + + +class GridCompositor: + """Facade that delegates to GridLayout or PresenterLayout.""" + + def __init__(self, ffmpeg: FFmpegRunner): + self.ffmpeg = ffmpeg + self.grid = GridLayout(ffmpeg) + self.presenter = PresenterLayout(ffmpeg) + + # Expose GridLayout helpers for backward compat + compute_layout = staticmethod(GridLayout.compute_layout) + even = staticmethod(_even) + + def composite(self, active_segments: List[Union[GridSource, tuple]], + duration: float, output_path: str, + target_w: int, target_h: int, + mix_all_audio: bool = False) -> str: + # Normalize to GridSource objects + sources = [] + for seg in active_segments: + if isinstance(seg, GridSource): + sources.append(seg) + elif len(seg) == 3: + sources.append(GridSource(path=seg[0], offset=seg[1], has_audio=seg[2])) + else: + sources.append(GridSource(path=seg[0], offset=seg[1])) + return self.grid.composite(sources, duration, output_path, + target_w, target_h, mix_all_audio) + + def presenter_composite(self, main: GridSource, + overlay: Optional[GridSource], + extra_audio: List[GridSource], + duration: float, output_path: str, + target_w: int, target_h: int, + mix_all_audio: bool = True) -> str: + return self.presenter.composite(main, overlay, extra_audio, + duration, output_path, + target_w, target_h) diff --git a/tests/test_grid.py b/tests/test_grid.py index e6e2c5d..1aa62ba 100644 --- a/tests/test_grid.py +++ b/tests/test_grid.py @@ -1,42 +1,192 @@ import os -from mtslinker.grid import GridCompositor +import subprocess +from mtslinker.grid import GridCompositor, GridLayout, PresenterLayout, _even, _build_audio_filter +from mtslinker.timeline import GridSource from tests.conftest import make_test_video +# --- GridLayout unit tests --- + def test_compute_layout_single(ffmpeg): - grid = GridCompositor(ffmpeg) - assert grid.compute_layout(1) == (1, 1) + assert GridLayout.compute_layout(1) == (1, 1) def test_compute_layout_two(ffmpeg): - grid = GridCompositor(ffmpeg) - assert grid.compute_layout(2) == (2, 1) + assert GridLayout.compute_layout(2) == (2, 1) def test_compute_layout_four(ffmpeg): - grid = GridCompositor(ffmpeg) - assert grid.compute_layout(4) == (2, 2) + assert GridLayout.compute_layout(4) == (2, 2) def test_compute_layout_five(ffmpeg): - grid = GridCompositor(ffmpeg) - assert grid.compute_layout(5) == (3, 2) + assert GridLayout.compute_layout(5) == (3, 2) -def test_even(ffmpeg): - grid = GridCompositor(ffmpeg) - assert grid.even(640) == 640 - assert grid.even(641) == 640 - assert grid.even(1) == 0 +def test_even(): + assert _even(640) == 640 + assert _even(641) == 640 + assert _even(1) == 0 -def test_composite_two_webcams(ffmpeg, tmp_dir): - grid = GridCompositor(ffmpeg) +def test_grid_composite_two_webcams(ffmpeg, tmp_dir): + grid = GridLayout(ffmpeg) v1 = os.path.join(tmp_dir, 'v1.mp4') v2 = os.path.join(tmp_dir, 'v2.mp4') out = os.path.join(tmp_dir, 'grid.mp4') make_test_video(v1, duration=2) make_test_video(v2, duration=2) - grid.composite([(v1, 0), (v2, 0)], 2.0, out, 640, 360) + sources = [GridSource(v1, 0), GridSource(v2, 0)] + grid.composite(sources, 2.0, out, 640, 360) assert os.path.exists(out) assert os.path.getsize(out) > 0 + + +def test_grid_output_matches_target_resolution(ffmpeg, prober, tmp_dir): + """Grid output must be exactly target_w x target_h for concat safety.""" + grid = GridLayout(ffmpeg) + v1 = os.path.join(tmp_dir, 'v1.mp4') + v2 = os.path.join(tmp_dir, 'v2.mp4') + v3 = os.path.join(tmp_dir, 'v3.mp4') + out = os.path.join(tmp_dir, 'grid.mp4') + make_test_video(v1, duration=2, width=320, height=240) + make_test_video(v2, duration=2, width=640, height=480) + make_test_video(v3, duration=2, width=320, height=240) + sources = [GridSource(v1, 0), GridSource(v2, 0), GridSource(v3, 0)] + grid.composite(sources, 2.0, out, 1280, 720) + info = prober.probe_file(out) + assert info['width'] == 1280 + assert info['height'] == 720 + + +# --- PresenterLayout unit tests --- + +def test_presenter_main_only(ffmpeg, prober, tmp_dir): + """Presenter with only main source, no overlay.""" + presenter = PresenterLayout(ffmpeg) + v = os.path.join(tmp_dir, 'main.mp4') + out = os.path.join(tmp_dir, 'pres.mp4') + make_test_video(v, duration=3, with_audio=True) + main = GridSource(v, 0, has_audio=True) + presenter.composite(main, None, [], 3.0, out, 640, 360) + assert os.path.exists(out) + info = prober.probe_file(out) + assert info['width'] == 640 + assert info['height'] == 360 + assert info['has_audio'] + + +def test_presenter_with_pip(ffmpeg, prober, tmp_dir): + """Presenter with main + PIP overlay.""" + presenter = PresenterLayout(ffmpeg) + v_main = os.path.join(tmp_dir, 'main.mp4') + v_pip = os.path.join(tmp_dir, 'pip.mp4') + out = os.path.join(tmp_dir, 'pres.mp4') + make_test_video(v_main, duration=3, with_audio=True) + make_test_video(v_pip, duration=3, with_audio=True) + main = GridSource(v_main, 0, has_audio=True, is_admin=True) + overlay = GridSource(v_pip, 0, has_audio=True) + presenter.composite(main, overlay, [], 3.0, out, 640, 360) + assert os.path.exists(out) + info = prober.probe_file(out) + assert info['width'] == 640 + assert info['height'] == 360 + assert info['has_audio'] + + +def test_presenter_with_extra_audio(ffmpeg, prober, tmp_dir): + """Presenter with main + overlay + extra audio-only source.""" + presenter = PresenterLayout(ffmpeg) + v_main = os.path.join(tmp_dir, 'main.mp4') + v_pip = os.path.join(tmp_dir, 'pip.mp4') + v_extra = os.path.join(tmp_dir, 'extra.mp4') + out = os.path.join(tmp_dir, 'pres.mp4') + make_test_video(v_main, duration=3, with_audio=True) + make_test_video(v_pip, duration=3, with_audio=False) + make_test_video(v_extra, duration=3, with_audio=True) + main = GridSource(v_main, 0, has_audio=True, is_admin=True) + overlay = GridSource(v_pip, 0, has_audio=False) + extra = [GridSource(v_extra, 0, has_audio=True)] + presenter.composite(main, overlay, extra, 3.0, out, 640, 360) + assert os.path.exists(out) + info = prober.probe_file(out) + assert info['has_audio'] + + +def test_presenter_resolution_matches_target(ffmpeg, prober, tmp_dir): + """Presenter output must be exactly target_w x target_h.""" + presenter = PresenterLayout(ffmpeg) + # Source is 320x240 but target is 1280x720 + v_main = os.path.join(tmp_dir, 'main.mp4') + v_pip = os.path.join(tmp_dir, 'pip.mp4') + out = os.path.join(tmp_dir, 'pres.mp4') + make_test_video(v_main, duration=2, width=320, height=240, with_audio=True) + make_test_video(v_pip, duration=2, width=160, height=120, with_audio=True) + main = GridSource(v_main, 0, has_audio=True) + overlay = GridSource(v_pip, 0, has_audio=True) + presenter.composite(main, overlay, [], 2.0, out, 1280, 720) + info = prober.probe_file(out) + assert info['width'] == 1280 + assert info['height'] == 720 + + +# --- _build_audio_filter unit tests --- + +def test_build_audio_filter_no_audio(): + sources = [GridSource('a.mp4', 0, has_audio=False)] + filt, audio_map = _build_audio_filter(sources, 5.0) + assert filt == '' + assert audio_map == [] + + +def test_build_audio_filter_single(): + sources = [GridSource('a.mp4', 0, has_audio=True)] + filt, audio_map = _build_audio_filter(sources, 5.0) + assert '[aout]' in filt + assert audio_map == ['-map', '[aout]'] + + +def test_build_audio_filter_multi(): + sources = [ + GridSource('a.mp4', 0, has_audio=True), + GridSource('b.mp4', 0, has_audio=True), + ] + filt, audio_map = _build_audio_filter(sources, 5.0) + assert 'amix=inputs=2' in filt + assert audio_map == ['-map', '[aout]'] + + +def test_build_audio_filter_mixed(): + sources = [ + GridSource('a.mp4', 0, has_audio=True), + GridSource('b.mp4', 0, has_audio=False), + GridSource('c.mp4', 0, has_audio=True), + ] + filt, audio_map = _build_audio_filter(sources, 5.0) + assert 'amix=inputs=2' in filt + assert '[0:a]' in filt + assert '[2:a]' in filt + assert '[1:a]' not in filt + + +# --- GridCompositor facade backward compat --- + +def test_facade_composite(ffmpeg, tmp_dir): + """GridCompositor facade delegates to GridLayout.""" + comp = GridCompositor(ffmpeg) + v1 = os.path.join(tmp_dir, 'v1.mp4') + v2 = os.path.join(tmp_dir, 'v2.mp4') + out = os.path.join(tmp_dir, 'grid.mp4') + make_test_video(v1, duration=2) + make_test_video(v2, duration=2) + # Legacy tuple interface + comp.composite([(v1, 0), (v2, 0)], 2.0, out, 640, 360) + assert os.path.exists(out) + + +def test_facade_compute_layout(): + assert GridCompositor.compute_layout(4) == (2, 2) + + +def test_facade_even(): + assert GridCompositor.even(641) == 640 From 2b669fc19bdc67731f8df3dd46193e7280d26b48 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 13 Apr 2026 02:23:21 +0000 Subject: [PATCH 59/65] Fix audio overlay crash: handle missing audio and sample rate mismatch The final amix step assumed the video always has an audio stream and that it matches the mixed audio's 44100 Hz sample rate. HLS sources come in at 48000 Hz, causing "Invalid argument" in the filter graph. Now checks for audio presence and resamples before mixing. --- mtslinker/audio.py | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/mtslinker/audio.py b/mtslinker/audio.py index 0f2c18f..6d2c1ab 100644 --- a/mtslinker/audio.py +++ b/mtslinker/audio.py @@ -145,18 +145,31 @@ def merge(self, video_path: str, # Step 3: Overlay mixed audio onto video logging.info('Overlaying mixed audio onto video...') - self.ffmpeg.run( - [ - 'ffmpeg', '-y', '-v', 'warning', - '-i', video_path, - '-i', mixed_audio_path, - '-filter_complex', - '[0:a][1:a]amix=inputs=2:duration=first:normalize=0[aout]', - '-map', '0:v', '-map', '[aout]', - '-c:v', 'copy', - '-c:a', 'aac', '-b:a', '192k', - output_path, - ], - description='overlay mixed audio onto video', - ) + video_has_audio = self.prober.has_audio(video_path) + if video_has_audio: + # Mix video's audio with merged audio, normalize sample rates + filter_graph = ( + '[0:a]aresample=44100[va];' + '[va][1:a]amix=inputs=2:duration=first:normalize=0[aout]' + ) + audio_map = ['-map', '[aout]'] + else: + # Video has no audio, just use merged audio + audio_map = ['-map', '1:a'] + filter_graph = None + + cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', video_path, + '-i', mixed_audio_path, + ] + if filter_graph: + cmd.extend(['-filter_complex', filter_graph]) + cmd.extend([ + '-map', '0:v', *audio_map, + '-c:v', 'copy', + '-c:a', 'aac', '-b:a', '192k', + output_path, + ]) + self.ffmpeg.run(cmd, description='overlay mixed audio onto video') return output_path From 480a6b37d48e0543357a72cb447d8d584c880755 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 13 Apr 2026 07:43:20 +0000 Subject: [PATCH 60/65] Detect screen shares from API screensharing events, not audio heuristic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old heuristic (has_video && !has_audio = screenshare) was wrong — it matched webcams with muted mics. The API provides explicit stream.screensharing data on mediasession.add events. Now uses that to correctly identify screen share streams for presenter layout. --- mtslinker/processor.py | 2 +- mtslinker/timeline.py | 6 ++++++ tests/test_timeline.py | 34 ++++++++++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 6826438..4210405 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -105,7 +105,7 @@ def analyze(self, total_duration, downloaded_files, directory, 'remaining': tw.duration, 'has_audio': stream.has_audio and f['has_audio'], 'is_admin': is_admin, - 'is_screenshare': bool(stream.has_video and not stream.has_audio), + 'is_screenshare': stream.is_screenshare, }) if not sources: segments.append({ diff --git a/mtslinker/timeline.py b/mtslinker/timeline.py index c863ad2..2b78f7f 100644 --- a/mtslinker/timeline.py +++ b/mtslinker/timeline.py @@ -19,6 +19,7 @@ class MediaSession: conf_id: Optional[str] = None has_audio: bool = True has_video: bool = True + is_screenshare: bool = False user_id: Optional[str] = None file_path: Optional[str] = None # set after download @@ -195,11 +196,15 @@ def _extract_sessions(self, json_data: dict): if not ms_id: continue conf_id = None + is_screenshare = False stream = data.get('stream', {}) if isinstance(stream, dict): conf = stream.get('conference', {}) if isinstance(conf, dict): conf_id = conf.get('id') + ss = stream.get('screensharing', {}) + if isinstance(ss, dict) and ss.get('id'): + is_screenshare = True props = conf_props.get(conf_id, {}) self.sessions[ms_id] = MediaSession( @@ -209,6 +214,7 @@ def _extract_sessions(self, json_data: dict): conf_id=conf_id, has_audio=props.get('has_audio', True), has_video=props.get('has_video', True), + is_screenshare=is_screenshare, user_id=conf_users.get(conf_id), ) diff --git a/tests/test_timeline.py b/tests/test_timeline.py index 2f34c1e..4956428 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -5,10 +5,15 @@ def _make_json(events, duration=100.0): return {'duration': duration, 'eventLogs': events} -def _ms_add(ms_id, url, time, conf_id=None): +def _ms_add(ms_id, url, time, conf_id=None, screenshare_id=None): data = {'id': ms_id, 'url': url} + stream = {} if conf_id: - data['stream'] = {'conference': {'id': conf_id}} + stream['conference'] = {'id': conf_id} + if screenshare_id: + stream['screensharing'] = {'id': screenshare_id} + if stream: + data['stream'] = stream return {'module': 'mediasession.add', 'relativeTime': time, 'data': data} @@ -176,6 +181,31 @@ def test_fallback_url_capture(): assert any('fallback' in sid for sid in tl.sessions) +def test_screenshare_detection(): + """mediasession with stream.screensharing should be flagged.""" + json_data = _make_json([ + _ms_add('ms1', 'http://webcam.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + _ms_add('ms2', 'http://screen.mp4', 10.0, screenshare_id=99), + _ms_update('ms2', 40.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + assert tl.sessions['ms1'].is_screenshare is False + assert tl.sessions['ms2'].is_screenshare is True + + +def test_screenshare_not_detected_without_id(): + """Empty screensharing dict should not flag as screenshare.""" + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 50.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + assert tl.sessions['ms1'].is_screenshare is False + + def test_time_window_properties(): w = TimeWindow(start_time=10.0, duration=5.0, streams=[ MediaSession(id='1', url='a', start_time=0, has_video=True, has_audio=False), From bc9b12a79ef417de3a2f0bab0fe8a569530c0c33 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Mon, 18 May 2026 19:55:57 +0000 Subject: [PATCH 61/65] Fix OOM in ensure_audio: bound silent anullsrc to video duration anullsrc is infinite; with -c:v copy the video muxes faster than realtime so -shortest alone lets ffmpeg's interleaving buffer grow until av_interleaved_write_frame fails with Cannot allocate memory. Probe the input duration and pass -t to bound the silent track, falling back to -shortest-only when the probe yields no duration. Strengthen test_ensure_audio_adds_silent to assert the output has an audio stream and that the -t bound does not truncate the video. --- mtslinker/segments.py | 7 +++++++ tests/test_segments.py | 8 +++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mtslinker/segments.py b/mtslinker/segments.py index cfccca2..c307563 100644 --- a/mtslinker/segments.py +++ b/mtslinker/segments.py @@ -37,11 +37,18 @@ def ensure_audio(self, input_path: str, output_path: str) -> str: ) if has_audio: return input_path + # anullsrc is an infinite source; with -c:v copy the video is muxed + # faster than realtime, so -shortest alone lets the interleaving + # buffer grow until ffmpeg dies with "Cannot allocate memory". + # Bound the silent audio to the measured video duration. + duration = self.prober.get_duration(input_path) + bound = ['-t', str(duration)] if duration > 0 else [] self.ffmpeg.run( [ 'ffmpeg', '-y', '-v', 'error', '-i', input_path, '-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo', + *bound, '-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k', '-shortest', output_path, diff --git a/tests/test_segments.py b/tests/test_segments.py index fc2c881..7396633 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -18,12 +18,18 @@ def test_ensure_audio_keeps_existing(segments, tmp_dir): def test_ensure_audio_adds_silent(segments, tmp_dir): + from mtslinker.prober import MediaProber vid = os.path.join(tmp_dir, 'without.mp4') out = os.path.join(tmp_dir, 'out.mp4') - make_test_video(vid, with_audio=False) + make_test_video(vid, with_audio=False, duration=5) result = segments.ensure_audio(vid, out) assert result == out assert os.path.exists(out) + prober = MediaProber() + # Silent audio stream must actually be present... + assert prober.has_audio(out) + # ...and the -t bound must not truncate the video. + assert abs(prober.get_duration(out) - prober.get_duration(vid)) < 0.5 def test_normalize(segments, tmp_dir): From 804d9bd164011c383351f04486e152532702cc0e Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Tue, 19 May 2026 18:08:53 +0000 Subject: [PATCH 62/65] Fix ensure_audio OOM: cap muxer interleave buffer, finite anullsrc The bc9b12a -t output bound did not help: with -c:v copy the muxer receives all video packets at once while anullsrc audio is generated lazily, so it buffers the whole copied segment in RAM to interleave, dying with av_interleaved_write_frame: Cannot allocate memory on long segments. Add -max_interleave_delta 0 so packets are written without buffering, and make anullsrc a finite input (-t before -i) so the fallback path is bounded too. Add regression tests asserting the command keeps both safeguards; the existing real-ffmpeg test uses a short clip and cannot trigger the length-proportional OOM. --- mtslinker/segments.py | 22 +++++++++----- tests/test_segments.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/mtslinker/segments.py b/mtslinker/segments.py index c307563..e1594d2 100644 --- a/mtslinker/segments.py +++ b/mtslinker/segments.py @@ -37,20 +37,26 @@ def ensure_audio(self, input_path: str, output_path: str) -> str: ) if has_audio: return input_path - # anullsrc is an infinite source; with -c:v copy the video is muxed - # faster than realtime, so -shortest alone lets the interleaving - # buffer grow until ffmpeg dies with "Cannot allocate memory". - # Bound the silent audio to the measured video duration. + # With -c:v copy the muxer gets every video packet almost instantly, + # while anullsrc audio is produced lazily. To interleave correctly the + # muxer buffers the copied video in RAM waiting for audio, which on a + # long segment grows until "av_interleaved_write_frame: Cannot allocate + # memory". -max_interleave_delta 0 makes it write packets immediately + # instead of buffering to interleave. We also bound the anullsrc input + # itself to the measured duration so it is finite, not infinite. duration = self.prober.get_duration(input_path) - bound = ['-t', str(duration)] if duration > 0 else [] + silent_in = ( + ['-f', 'lavfi', '-t', str(duration), '-i', 'anullsrc=r=44100:cl=stereo'] + if duration > 0 + else ['-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo'] + ) self.ffmpeg.run( [ 'ffmpeg', '-y', '-v', 'error', '-i', input_path, - '-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo', - *bound, + *silent_in, '-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k', - '-shortest', + '-shortest', '-max_interleave_delta', '0', output_path, ], description='add silent audio stream', diff --git a/tests/test_segments.py b/tests/test_segments.py index 7396633..90c8de6 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -1,5 +1,71 @@ import os from tests.conftest import make_test_video +from mtslinker.segments import SegmentBuilder + + +class _SpyFFmpeg: + """Captures the last ffmpeg command instead of running it.""" + + def __init__(self): + self.cmd = None + + def run(self, cmd, description='ffmpeg'): + self.cmd = cmd + + +class _FakeProber: + """No audio in input, fixed duration — exercises the silent-audio path.""" + + def __init__(self, duration): + self._duration = duration + + def probe_streams(self, path): + return {'streams': [{'codec_type': 'video'}]} + + def get_duration(self, path): + return self._duration + + +def _flag_index(cmd, flag): + return cmd.index(flag) + + +def test_ensure_audio_guards_against_interleave_oom(): + """The silent-audio mux must not OOM via the muxer interleave buffer. + + Regression guard for av_interleaved_write_frame "Cannot allocate + memory": with -c:v copy + a lazily-generated anullsrc, ffmpeg buffers + the whole copied video in RAM unless we (a) cap the interleave buffer + and (b) make the silent input finite. A small test clip never triggers + the real OOM, so we assert the *command* keeps both safeguards. + """ + spy = _SpyFFmpeg() + builder = SegmentBuilder(spy, _FakeProber(duration=1234.0)) + builder.ensure_audio('in.mp4', 'out.mp4') + cmd = spy.cmd + + # (a) muxer must not buffer to interleave + assert '-max_interleave_delta' in cmd + assert cmd[cmd.index('-max_interleave_delta') + 1] == '0' + + # (b) anullsrc must be a finite input: -t before its -i, not after + anull = cmd.index('anullsrc=r=44100:cl=stereo') + t_idx = cmd.index('-t') + assert t_idx < anull, '-t must bound the anullsrc *input*, not the output' + assert cmd[t_idx + 1] == '1234.0' + + # correctness not regressed: video still stream-copied + assert cmd[cmd.index('-c:v') + 1] == 'copy' + + +def test_ensure_audio_safe_when_duration_unknown(): + """Probe miss (duration 0) must still cap the interleave buffer.""" + spy = _SpyFFmpeg() + builder = SegmentBuilder(spy, _FakeProber(duration=0.0)) + builder.ensure_audio('in.mp4', 'out.mp4') + cmd = spy.cmd + assert '-max_interleave_delta' in cmd + assert cmd[cmd.index('-max_interleave_delta') + 1] == '0' def test_generate_black(segments, tmp_dir): From dbf831af84b8114b530799cb3511ffb261ecf570 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 20 May 2026 05:59:52 +0000 Subject: [PATCH 63/65] Fail loudly on normalize past EOF; reject empty input in ensure_audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the av_interleaved_write_frame OOM: when a manifest segment has source_offset > source file duration (planner asks normalize to seek past end-of-source), ffmpeg decodes zero frames, writes a valid 262-byte empty container, and exits 0. ensure_audio then sees duration=0, takes the unbounded anullsrc branch with -c:v copy, and since no video packets ever arrive, -shortest never fires — the muxer buffers silent audio forever until RAM+swap are exhausted. The previous fix tuned ffmpeg muxer flags but never closed the duration=0 vector that bypassed them. - normalize(): after ffmpeg.run, probe output and raise CalledProcessError if duration <= 0. - ensure_audio(): raise CalledProcessError on non-positive-duration input and remove the unbounded anullsrc fallback entirely (it had no safe semantics). - processor.execute() video branch: wrap normalize+ensure_audio in the same try/except → generate_black fallback already used by the grid and presenter branches; a seek-past-EOF segment becomes a correctly-sized black gap instead of crashing the whole webinar. Reproduction (segment 73, source duration 67.24s, source_offset 104.17s): before: normalize produces 262B duration-0 file exit 0; ensure_audio runs unbounded and hits ENOMEM. After: normalize raises with a clear message naming the input and seek offset; processor falls back to black. --- mtslinker/processor.py | 28 +++++++++++++++------- mtslinker/segments.py | 42 ++++++++++++++++++++++---------- tests/test_segments.py | 54 +++++++++++++++++++++++++++++++++++++----- 3 files changed, 97 insertions(+), 27 deletions(-) diff --git a/mtslinker/processor.py b/mtslinker/processor.py index 4210405..fc6052d 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -383,14 +383,26 @@ def execute(self, manifest): norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') source_offset = seg.get('source_offset', 0) planned_dur = seg.get('planned_duration', 0) - self.segments.normalize( - seg['source_path'], norm_path, - target_w, target_h, target_pix_fmt, - max_duration=planned_dur, - seek=source_offset, - ) - with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') - final_seg = self.segments.ensure_audio(norm_path, with_audio_path) + try: + self.segments.normalize( + seg['source_path'], norm_path, + target_w, target_h, target_pix_fmt, + max_duration=planned_dur, + seek=source_offset, + ) + with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') + final_seg = self.segments.ensure_audio( + norm_path, with_audio_path) + except subprocess.CalledProcessError: + logging.warning( + f'Video segment {i} failed, using black gap') + gap_dur = planned_dur or seg.get('source_duration', 0) + gap_path = os.path.join(tmp_dir, f'vidfail_{i}.mp4') + self.segments.generate_black(gap_path, gap_dur, + target_w, target_h, + target_pix_fmt) + concat_segments.append(gap_path) + continue actual_dur = self.prober.get_duration(final_seg) if planned_dur > 0 and actual_dur < planned_dur - 1.0: diff --git a/mtslinker/segments.py b/mtslinker/segments.py index e1594d2..95a4286 100644 --- a/mtslinker/segments.py +++ b/mtslinker/segments.py @@ -1,4 +1,5 @@ import os +import subprocess from mtslinker.ffmpeg import FFmpegRunner from mtslinker.prober import MediaProber @@ -37,24 +38,28 @@ def ensure_audio(self, input_path: str, output_path: str) -> str: ) if has_audio: return input_path - # With -c:v copy the muxer gets every video packet almost instantly, - # while anullsrc audio is produced lazily. To interleave correctly the - # muxer buffers the copied video in RAM waiting for audio, which on a - # long segment grows until "av_interleaved_write_frame: Cannot allocate - # memory". -max_interleave_delta 0 makes it write packets immediately - # instead of buffering to interleave. We also bound the anullsrc input - # itself to the measured duration so it is finite, not infinite. + # anullsrc MUST be bounded by an input-side -t equal to the video + # length. With an unbounded anullsrc and -c:v copy, if the input has + # no decodable video the muxer waits forever for a video packet to + # interleave against, buffering silent audio without bound until + # "av_interleaved_write_frame: Cannot allocate memory" (observed in + # production on a normalize() output that seeked past end-of-source). + # A zero/unknown duration means the input is empty or corrupt, so + # there is no safe silent track to synthesize: fail loudly (the + # callers fall back to a black gap) instead of running unbounded. duration = self.prober.get_duration(input_path) - silent_in = ( - ['-f', 'lavfi', '-t', str(duration), '-i', 'anullsrc=r=44100:cl=stereo'] - if duration > 0 - else ['-f', 'lavfi', '-i', 'anullsrc=r=44100:cl=stereo'] - ) + if duration <= 0: + raise subprocess.CalledProcessError( + 1, 'ensure_audio', output=None, + stderr=(f'refusing silent-audio mux: {input_path} has no ' + f'positive duration (empty or corrupt input)'), + ) self.ffmpeg.run( [ 'ffmpeg', '-y', '-v', 'error', '-i', input_path, - *silent_in, + '-f', 'lavfi', '-t', str(duration), + '-i', 'anullsrc=r=44100:cl=stereo', '-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k', '-shortest', '-max_interleave_delta', '0', output_path, @@ -84,6 +89,17 @@ def normalize(self, input_path: str, output_path: str, ], description=f'normalize segment {os.path.basename(input_path)}', ) + # ffmpeg exits 0 even when -ss seeks past end-of-source and zero + # frames are decoded, leaving a tiny but valid empty container. + # Treat a zero-duration normalize as a failure so the caller falls + # back to a black gap instead of feeding an empty file into + # ensure_audio (whose muxer would then OOM). + if self.prober.get_duration(output_path) <= 0: + raise subprocess.CalledProcessError( + 1, 'normalize', output=None, + stderr=(f'normalize produced empty output for {input_path} ' + f'(seek {seek}s past end-of-source?)'), + ) return output_path diff --git a/tests/test_segments.py b/tests/test_segments.py index 90c8de6..84d8836 100644 --- a/tests/test_segments.py +++ b/tests/test_segments.py @@ -1,4 +1,8 @@ import os +import subprocess + +import pytest + from tests.conftest import make_test_video from mtslinker.segments import SegmentBuilder @@ -12,6 +16,9 @@ def __init__(self): def run(self, cmd, description='ffmpeg'): self.cmd = cmd + def get_video_encoder_fast(self): + return ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23'] + class _FakeProber: """No audio in input, fixed duration — exercises the silent-audio path.""" @@ -58,14 +65,49 @@ def test_ensure_audio_guards_against_interleave_oom(): assert cmd[cmd.index('-c:v') + 1] == 'copy' -def test_ensure_audio_safe_when_duration_unknown(): - """Probe miss (duration 0) must still cap the interleave buffer.""" +def test_ensure_audio_rejects_zero_duration_input(): + """Empty/corrupt input (duration <= 0) must fail loudly, not run ffmpeg. + + Root cause of the production OOM: normalize() seeked past end-of-source + and produced a 262-byte, duration-0 file; feeding it to ensure_audio ran + an unbounded anullsrc + -c:v copy mux that buffered silent audio forever + until "av_interleaved_write_frame: Cannot allocate memory". ensure_audio + must refuse a non-positive-duration input (raising the same + CalledProcessError the callers' black-gap fallback catches) and must + never invoke ffmpeg in that case. + """ spy = _SpyFFmpeg() builder = SegmentBuilder(spy, _FakeProber(duration=0.0)) - builder.ensure_audio('in.mp4', 'out.mp4') - cmd = spy.cmd - assert '-max_interleave_delta' in cmd - assert cmd[cmd.index('-max_interleave_delta') + 1] == '0' + with pytest.raises(subprocess.CalledProcessError): + builder.ensure_audio('in.mp4', 'out.mp4') + assert spy.cmd is None, 'ffmpeg must not run on a zero-duration input' + + +def test_normalize_fails_loudly_on_empty_output(): + """Seek past end-of-source: ffmpeg exits 0 with an empty container. + + normalize() must detect the zero-duration output and raise + CalledProcessError so the processor falls back to a black gap instead + of handing an empty file to ensure_audio (the OOM trigger). + """ + spy = _SpyFFmpeg() + builder = SegmentBuilder(spy, _FakeProber(duration=0.0)) + with pytest.raises(subprocess.CalledProcessError): + builder.normalize('in.mp4', 'out.mp4', 640, 360, 'yuv420p', + max_duration=90.0, seek=104.0) + # ffmpeg *was* attempted (the emptiness is only knowable post-run)... + assert spy.cmd is not None + # ...and it really was the normalize seek that ran. + assert spy.cmd[spy.cmd.index('-ss') + 1] == '104.0' + + +def test_normalize_succeeds_when_output_nonempty(): + """A normal (non-empty) normalize must still return the output path.""" + spy = _SpyFFmpeg() + builder = SegmentBuilder(spy, _FakeProber(duration=90.0)) + result = builder.normalize('in.mp4', 'out.mp4', 640, 360, 'yuv420p', + max_duration=90.0) + assert result == 'out.mp4' def test_generate_black(segments, tmp_dir): From 1e5e796860ca020101e2bed95b954a1a0ff4aa1a Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 20 May 2026 06:00:44 +0000 Subject: [PATCH 64/65] Stop tracking tests/__pycache__; ignore *.pyc and *.egg-info/ The pycache was accidentally added before __pycache__/ was in .gitignore, so the rule never applied (gitignore is bypassed for already-tracked files). Untrack them and add *.pyc / *.egg-info/ so they stay out. --- .gitignore | 2 ++ tests/__pycache__/__init__.cpython-311.pyc | Bin 147 -> 0 bytes .../conftest.cpython-311-pytest-9.0.3.pyc | Bin 2701 -> 0 bytes ..._audio_pipeline.cpython-311-pytest-9.0.3.pyc | Bin 17493 -> 0 bytes .../test_ffmpeg.cpython-311-pytest-9.0.3.pyc | Bin 7805 -> 0 bytes .../test_grid.cpython-311-pytest-9.0.3.pyc | Bin 9633 -> 0 bytes .../test_prober.cpython-311-pytest-9.0.3.pyc | Bin 17714 -> 0 bytes .../test_processor.cpython-311-pytest-9.0.3.pyc | Bin 12284 -> 0 bytes .../test_segments.cpython-311-pytest-9.0.3.pyc | Bin 10455 -> 0 bytes 9 files changed, 2 insertions(+) delete mode 100644 tests/__pycache__/__init__.cpython-311.pyc delete mode 100644 tests/__pycache__/conftest.cpython-311-pytest-9.0.3.pyc delete mode 100644 tests/__pycache__/test_audio_pipeline.cpython-311-pytest-9.0.3.pyc delete mode 100644 tests/__pycache__/test_ffmpeg.cpython-311-pytest-9.0.3.pyc delete mode 100644 tests/__pycache__/test_grid.cpython-311-pytest-9.0.3.pyc delete mode 100644 tests/__pycache__/test_prober.cpython-311-pytest-9.0.3.pyc delete mode 100644 tests/__pycache__/test_processor.cpython-311-pytest-9.0.3.pyc delete mode 100644 tests/__pycache__/test_segments.cpython-311-pytest-9.0.3.pyc diff --git a/.gitignore b/.gitignore index eab9fc4..88c9691 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ *.mp3 !get_sessionId.mp4 __pycache__/ +*.pyc +*.egg-info/ diff --git a/tests/__pycache__/__init__.cpython-311.pyc b/tests/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 70d04f33542864c0176377a23830ac6b635bba82..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 147 zcmZ3^%ge<81QE+_WP<3&AOZ#$p^VRLK*n^26oz01O-8?!3`I;p{%4TnFE#z5{QMIA z+>+v)%)IQ>BHgt7qHO(=)Z&t2{rLFIyv&mLc)fzkUmP~M`6;D2sdh!IKy4s{i}``X R2WCb_#t#fIqKFwN1_0@>Al(1} diff --git a/tests/__pycache__/conftest.cpython-311-pytest-9.0.3.pyc b/tests/__pycache__/conftest.cpython-311-pytest-9.0.3.pyc deleted file mode 100644 index 8fd76a21a4ca9ffb42121b706d200b74aef898eb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2701 zcmbVN%}*Og6rWx1+Pmuy3;_a!q@-yPCIV7Q(iYWG+Cw6>3atu_D(7Gs&)96Rz3%K9 zg5^j)a7d&a$O)t<66%2#C5QY8JyktA5(jIgNRg^Y<>rW-P!D}?);2gam+s8`-kbMk zcIJJ|%&+0FLEtOw)Xg!KkiW3g`s5~a@DDJXgc2&{NJ0`+&im4`MDj{Pl{}3<;TM`f zA|R-i&;->Jx}ZkF04nE#`A{Mx=J;~qd?XPOyprq6M-x$rFyl^lYXekWCp*x2+7ggN zPlw)rLVv15A2^}!?a*r{^nEl+!|PI_pLWwOpr@%cPhvg$xP!S}$qik)`1J zS07%jDT5pZ|94j$Q-wp85FSLpfAKJY>n{oi@F;XJdhK=lAH*NVYs!GPZY!Z+1W}*~ zZ+r+$r#C93BJIG~r|t3c#9KQ6V`H)~PpoC*h>;O*6PoS$-sU|Z?T2O~gXRxzjc$*= z2*&n;v6>P)=>*)&1!xBp;9DZSv!OffP<00C9;uL9WK)uegI;NVchD^HRp7)tlRHhp zY=?~U%`LPkL~Yzq z)9iqgFD9wU`4Gs?K(jvw_!|mXu0{8>ffw4)o;Fnds%`|UH?X`^jP;uv=eK8S%Ge8K zY)=^j#;d|N`)I+!{xw)-zX5Fem~{kWECROy%qGc_dum1IAu3Us`lwQo9+M|BRjHo_ zvVnW*@iag3Bh{!*4H|6D3RQfM2@O{izzF_8r@zHa)pf9_D8$hc}QwURM`HOs@cSzz%DXFr!%A-xu#nYlDeOFZS6Rv~_< zOr&R1t}m5Jm-PvI$>MmFM1sNOHE1iYDxy;krLCOBMq-4USjEGP#AV; z5spxr*|zKBr9#Z_D#etuvMIzdt+v+cWbq-hwra$DYbLo{q`dO)aSQ6`n%>)z=qGK z&%QEx8!FI-pBT|bz~g$(w79dQEpF`77I*mzkJHG&@b*k2=vV!32*6u}MkjbVTZB|d zx6=WP!H(L68;(T1Fp6RZ=hoqY&UeRxRz}929w1#@m{5Dn{hmalbo$`(_yaO7P*0z%z()v zeN)_(Et{XlB_ssMX!odC*i6nb<$mH)jH{_ ziqh7;BC%R$t&_1@XRVWoT4#lb%G&z5hAc@#4Fd4SN91TrptY5Qs2n_|9BdpZ$5As z_RM(cudgB&y&j?Bp9y$aNpL1eOF}at`1O^;mB>t_5}k?CaI;8j;vBD%hjU1BdeAw3{z5+x~xpz!`V2M^BT!Y;ewpUCi53dl3cShX!ALb5j%FQ zGB3}asaLBqfqe8;St=D?Bek-&c(&c9mxpAI5B%pWMUwAUO4>azP`a)X&b7T2WRM!$;~>BcP~o z+X)oRmrE)>s*2&CE!PUF;ioM4QAue&=QG6lf_lyf5Lv00pGPVmmc!)yWh0uOli|UpvqQ$Aof!Nn*MTewU&Ys6eRnyW zZceubTlIGJru6x-TZcQLl!@Kq2-8WyQm+>M}D z(1KdPHNub$q){zy&S&Bgp4o)+UORKvGYijc6rNk_0fXO1*xlE=_PwM!)@wjZ z!p^(l#>T$g8Q+ZNp}}6~Y9sZSW7y=XpVm_C0jqucuNHt2Wntg-XkINX`EHE4_g%-T zO*`kQF+f-Cz$33(K=wUlo1bvi4miiO>Z*0&VAXE^-mKd3Mg~@3W&z$ojX`Zd8?>$5 zJC0HIc0#stw;=laG^gLgT{z=>LZq#q_}IQHJtaM@h0R@K%_Hm@Yn&}PbGBV0(-@Re zpdf5pcmY+hY|+%k@?1e-T|^q&u)^+~l8ZEq0B9PZ zh8nb6)K{*HhHw7z6p~#?b|cwy>bmD)0Im)${kmsi2pfP#vy4N#h4#!wKwY@*CA(nc zWE#lBgmb)P4>0dxF|_G92+a*x%^l26m_u_^+6zs?AoebVL5(7X$;x!$-aJer^s)__ zF~YM%eys%xI76XcL)HQ&=ULd$8v>mjoVy8n zUDGg6IUl>6-2ldsOfT54={YvM=W-dtTT{p;1c~Qsr7FQ&O;Ch2!u%mJ!u(Dcu|xBh z^JhV;S4&mJ2$X9@czTQoaH&C|K#X;H_RQspJWuj-<=R=ej%uMI8{xcCn3a)jBwnNs z1lTr?5>g{ZCJ&ljrJ&~J_vg!nDz$o0?afH%p@%YU3;Am%`G6Yyqe^)rjW^86Hool=4(Uwr9qIJyE;%rswWr?06Uak?u`>*6#d z?wZVcN8HmD_vqptNIUEJEG6pJj*(q7JE5F3Mopjj7+XE_G?wHjv)*%%zui8_V=HCPd= z0cq9-ZwWa3yqg?p?dvAD=*cZGJENOA;-;m2t?5n_Q|!DWZqltZ<4}aSscRK51kVD9 zahCRRULS*lT5wx?9YC7(!71ZFebxl<8QxxtgO}G1cf{c)X%^vUXrrK;GibqAl4>O`w68C%g4d+_F8s4q~VFqtoi8ak9YiW zN9)qHXm?;*ADCV|iBJE@YhyQ+j`)06d|nryhs0O$b*-WHP&d9!k8fK%b~l!|`tpab zTzRFnw*5>uHmS!Z7mqHBk>9=Y!7I&mt+%g@ch*mKMxVX8_VcZs*G@0K(iP9>;+c+k z=Bw0b>u5K%Sx;?VJh8k!+Yz%%`)CUDwB!9~$Ac9j{bUeUy=yz(z zqZ;}8g=17BH}!B8T(73$O?`H=C8DU{!~>UGcAHWosx1A4l~0_N3#3$?19u)ka+3>* zp2|(t^B4&IRNtT~I(m2BAWs85Mz#Z`s<@?49wIxCHGyOj3DuN*pgNPCkUd5AB6A-Q z!-pE?PcZvD5>$N20VF>~@&b}WNL~bzOOcb9M)i@rf`n_2f`8kqtByR%AC3~72`Ey_xfs1*(ZX?Woama5Yx{TxrNE|Ln8Jd$4`DIhru zL_s5#OPOMd6m_iUZHg4$lsujBJiRGo9;)K?E8F4P(1Vz~`qqc*IQGX_>#cURo7$tN_AH)2>G4?m^i9yCK&;)Ti~Asf%oEo6JWKFZ!jg07#k;TKyLn9q=WNCL( zT6zHz?7Sn6=vJC>5CEyJRlpED3q_2x1lp-tw~xU=O^Jh8n?UU+lrk(I2gBQIas2YY zU`HHmZf<7a2ZousvL6!cydw_kR+@1T0I9B3zz{qOMT~>jVb<+qa8PqI2eCGR+D|BD zSUwJhx7V`cdAeWpMVk_C4-fwP+Yx=y{$n2C(Di~VLF{vKSOEBa1V@VB<-`yj(IFy6 zq1*`d^}%09;AHG~q&%(@JM0<*I4hLQUT(<$|Q`bni5JP>4q%_zFYj9`4 zy`9oh;NG^+M(^IO)l%TzUWdp&r_S&b-Vv9}bJaWI!oeLGc8s=n?_7?5X~cAIr=?M8 zy_PO9_x8Xe+!2>^ZpAy2f|Y=?n0(|}1Z3YVZg|33OghK5>MXi&FpC?$KeGs~sf`Oe zY0zMnTE~?vnE8vclttk#Tae^(;j-OVrJO6xs*?+08lXr8Tzm?cfV&W-aBx;zCX4!V zz!7uZhmMq;{5n>V)zaR1OD=Z8Xm1iISv$GV$o`CCd;YgXHH*Xp;(A*VPK5*S{!Uoz~rYlL9Aur zTDqHhMo&Gnc;c(X`qt0eFSg(7CUSZrxA^jxKX~K!^2o-HxN+&Z)|O5bQ|!DWZq%(b z<4}aSv1=7D1kVD9ah9Irygmj8wYG2&@F30l;FNKoK5K$uU_ka-9Ly5^VtWGC92MKq zkNq($8pf3jk{vs z;S?2VptmT+G?4~++r@NoWszEPiD^=z7mB#DDDCWVyT)C&-Y;Ah_|q0^$~x0Wh=ixGfY@W&1enO=DbXnyCT-hhB>3=-7> zriK}kq9A7nw6K|n;84&wBcO3|9m*b94T~DWO+n*~V;dlIyERUGEw1ZmF3k@@lW`rG zN6)P6T(C53+uk16Ox=1kNal82uU}W_F-YdlaO@t^t=}X~jWa5xAP_lf>NhbJ7o~YP z`otAa4=v`Lac2%R&H>OggQnIQ0X=5`Hl57E;ZqodQ!XulV6Lzacoa)2hv1yzIh5Mr ziy$d$iJ(+|hi_2ST3Lqes3?QXP7SBjL>;O^dzc~whL!WMyUId0X2 z1UG)#nN*C$D$EwzXJcgyyP=eEP-=D9Wo-=JS5x62)`Pug&VdG5n~Vd7VtXxD6??XS z=MTb-L9ygP3f_B8@#wqQ`C(sPQS<>@AXq>p(P??3AF*(;tl@``Sp1sk+zT23EdU>} zz&Ca(x23^ei`PYu6?VfFElF;XZ&f>m>th$#4Fjv#DO_(2DfomVbY1zXBXr?lguzut z=-Sz(kogfy&~Ec_)a|`JTyG=iU4b1MZGrwqz`ob_-8&GM;k_B6_9?&ScUk%P+W@|H z^n;ZzipW0uRO|_7)n&(Bbyi(CnAMnLW>&ZI#Z4<;L`r~-IbyyAzr8sQ!$$>SEes#A z1Ym|NndIx{Gfn4n0MS3-`HV^#Y0xHB*zE6)JJWk+ze%c~&n!H*VUX~`AW^JYIQSBn z`7UJV3iCaIqEZK!_qoDFaD=n3l|i$iUp0f$Ar~t2Gv{KhS}dRe@GRD+BE&PWAFbra zX71;mP-^pIGheXNk9`lF&>)1M;Sb`cc?kCb9>PryK@oFpuP93xi8&d*hc2jco{sDP z;NW%(6yz#)xd{oim_33wdlgw?J(VTsxbl`n!Q-~cAD{;+iTo>w`u$P_7}K8n@W7P= ztzdh!8=cUj6N^Wd#n>0(=xuSdb)-EGp}XMKozlgrjySb!`kBW}Kl2e53)~TbF@}_$ zcf>JWgxETyb`Hkqnmh)vF#yMSG-%g>;*~fC;};ze8-s&jL>vW7NV6id8*6}e3;@`P zerBl~8Py}BKl+)eH`(s^?Q<*lF_`Zd9N6EjpE!nj1>PQeUg1= zPf*UXv)M-~1l42s4;HKs*e98d9mOjn(c{uj6wr!b4D^^>)1X?-IF!=(ej1WO?cvP0 zg8Bhn-q!^xpHj`@Vdy}mCe_RGLGt%d4w{3YWPyS`#p78P-s}i(F5^D1%D?a;{kJTn zJJz`@Jl|RATo!U2=eaCw>p0J4;pvX^T&dPXM{u0?17T02kNza=5 h0w-J6c{be_IN2M)%AAZuJfoO3$w&z6_7ggT{|n449_s)A diff --git a/tests/__pycache__/test_ffmpeg.cpython-311-pytest-9.0.3.pyc b/tests/__pycache__/test_ffmpeg.cpython-311-pytest-9.0.3.pyc deleted file mode 100644 index 95959f79440a65b112f36d686b3511fb23e4a54b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7805 zcmeHMO>7&-72ZGoEk()lpJdl*Wjltw3M5%FB`a=Z*flJ)DC)X3QXsl)Y3@>`%s5Jp`!GgBry-_*nFoTaTnbA;O{o3iOnlTjyZt)c0nVqur&b$Zjt! zvZ5Z}y!qan+1Y(RGyGF3rAu(VzrLOSyDCZlz()I0Jm$sEfq5cXk|h_UIhlPcMK!C) zQc;_W$h;>y7iB%Mxfpyag?KSBmw-O?*V3GBX+V=!1Za;H1)8#AKzpq?&^}APBN>S= zakyKC?Dovg7AyArPphSpO|mkIy(rJrSVu2~T)TGR3}Yek&?_6#H` zU`gC{EfTa8Z2bz>_@P{v;a`QdmKNAMhBjYu^<(&)zFB?#G!|ZK^(_f}D-Le86V211 zQrR)&rSG1*P;eFn=`t&CY-v`HcJXrD2Sq^n!DV`m+6zEXAj zLUVNWYn4ZiO&v2s+khdt-R-lu?sE zoYBf7_as}9-DNYW?NBpU$jnpnZLo+|>E9p*Elb;){<(H+Q#;nw23p!cLmSvm_P3I! z8`{XnM}ahwr{Oy}f-L)NX(O#R?+F=18)>$CI7Bo7#U(3KLe?P=AQ+INIOOA_LI)tg zdtCSE}Q4}|`Xl=>f*2BAdWwKNFfpfrFaF4B3_iv8Eqxl&iXWU?Xe zC^It<+b{1E+e1Mb63GnDGHHYIz!=Hp33UvW#Nj0+nB6dEq?i1OWQr$}DMKeIMD-%+ zLvjcS7AoX0k|RitA~}ZSI1n!Yld~uoL~;^{KekT#8<%~7mb`(;ju4u+>|&+t*d>Qt z06-7?)2VJDwC8ZE=gjiW?!k2!!ra=so5|sZ=6$xb;Z~dXga`q)+3w*G(S$B8!LZ{g zcL)TC#nk{xC<)BkyF$hd`2+ysAAktnF98j%1VU~`7z^Q{NxlQ?w|8**-7D1hP*ciK zWkR@ElvrSS;)q}Z)p>gt1A7uy(W{6I2bO?_(5}6{a|;_V1I)DSs)d{uoZF8Rd>H3B zx8l|6LFYzi@UDP0bSicos8$E(cF={qD?9^m{dhfEQ)|(CY`0U7)nfEtf$?_NIZ=zj z&Phjfc%7}}0mI{ofNimY4*I%x1@{~zs3F#}y9TUk($Wq%pWu@c^9irht31)b9g6wZ z)Zl$}K}CZotLGz)y&Z8YW%btL-cByzeEr*z2wDoZyq%m1JGodL_V=)t=)1L}jbyJZ zZpf~ZFKx(6hcO9^8+0gNVzvJz4B4=W*ibN>k0ZGh*c`wvg5ZwuOMGQuW1?KeTyhc3 zfG*#Z0JFO(x!{dpKoYsilL*TT1-oQuJbPROQ)C>;IUs=)GR;%Sw4uBG^R{CyLKH8X zFu}5A6JIW2$s?3dL@ptdI9Ht|E;0%{&m6q0zvypF_;X5{d@F>VBqu;Z&I>Uw`QN6v zBD~5+$`g>mYWR*x!ex65q`o-?2X3}|pA zj)4gYV}AS>$=((E{`3daAY8oS6( z69~1HX%6DHSMLiIKmgB25R^**%i$c;ZB2w0F#BXv2VRMTmipghQ*a`71tRY&XH&0K zCc|KhyoF>035E_bhGYT>I=zu_k9WFwE<>Gz2B&tR%ZyK>-M5ikL4xt8qp%(A5?650 zy@Ja%FoI!=xWhj{9fj#>! zGE^vcL4`^iE8*=|BbII@YZ0$#i5}qiIaL1aS+ponv}oR*wdWSg+)MEhIBcYNXczMs zC}WbAO|v9l!NWX)&yJi10wu}BKHFQKf-wQ_?rhodD6wg^;1FCJl0<^LM5sMfQZ2dq zuA0xeLq$hu7aQY6$4#`wcCoC0z^yEEZgL7Ziod1$4kLP|rxOSy6IQr)wtjsl2v#r$Z@+>CCc{T8#pQ9k4|eoWZRA;&BM=Ve}s7PsTzzG>}NFi zDXmj_M>;EwOH_NHJXGJ49>}uv9@s}bl;hehI6%RPFDWDWrDJ2Ia>;&}r+7wT=JEx5 zv{*^I>fMY*ZUD%LxKVyq?nX%_Pi^Yzw0gHf%J_=7hi|~yxozF%O?Ko>W>HOMLr`20 zuQVMH?^h0fDyS0Mzx+wtBH z^~ZYS$UFS=*K;d+Gk&`jzuk!6ey;8UIsBhY^(&tNzp-xo%a<>^b@}BKo$wDQuSeC- zqS|#m_E|~>Y8-Oo?OzVqVUv5p$FL>UdOJx8>>#8$cf`RwVmyziv@Zxkh(|#Uj*5t9ohQY zA??I21-Od>)h=ADZmYu{bn@145BXE1Kn4r~0Tw7wAU6Z6`9uReZ`}ZX_+kw33@?-N-MCZujsj3ReiOlgrTQuLLVl zW(#{D9r&7}Xp3u2Nz%n)ND^$x4z=K9T`Goc*$%H$M-&xXF>6*Bn`6fswnz^>WGgG8 zE+cwH+5<-(z~z>Ot(eCP!kS-y50nRf!KPqd=4>lokuCE5B0MhA36~S~FlVuelz59B zP%ZxGfRfK;E6k&GhE12~P!#tGXRG z$Cn{2H!LHWNp0mfEVVd_*t>Bvt`>)8SBy@P!uw$-RM@(WWCF=uB=?X^pZ2Cq z(;yaYDv`h%g{?$lUnLkxFm5fttOg~_1s^Pn)r`peo$+~%YS6QA~w zcbwk1Me=#;R@O2z>D;>wc&;~yz{Fgt+$~K*5t9ur$B1JYJ-(eSt%&0kEwRJ`uK%5hAce~<*|mx zyGjipz}CGI2JuBGVw%!jUw1~~gP;!##URihYL-bgZ8#`aANvMb`#^&ylrp<+ZLg2P zT0wkU{sg=yTnxJCg|+5T+A=s_47#m@F8+4HpgRU%Y~)4X5q!hxKteR+aM&mBAn8Z) z9uR&IARmCrxzg?hx1Q$7O(>;?MzMMv$VH%n6D|T32K(mB@4IIQIAQYwl}77-5U5w% z1?ozEgOFR$$|bnSMWkNLV*|1II`dduLd2z4I7?*X>Kl*L^rk`XfcYPsBU1kX-Fagx zF7ezeE{U$VLTy)E@>^SR$=s@?IZb%`E3UA=;!^a;e|yE%vcisbTwz~4@4y()A$I<~ z753YF-m#9)TfKO{oAZ(X-{(ztRJ8e5J21N^ILgJv?O(s%CW^6X$v_t6c)DQ8;>I#ypNUkAy7s-#13?S*Wu!zz58Z|WqIILwZ!td+Zje_0j znmklq08*MRO_vuwpWB@Sp+Yrtm4z4D5QywNl!qD~&om`K09W@47{V8!h-q+#z^!{m z(F9F15JQv~f>LQNC{^4t)(I$Kj8B;c>@ZWHXSEsiVVxM*Ke=$rS?9QJ87M!h$pbW6 z9(_KuI|D)`fts00;)ON<96Jx?friI34I+T6dj$;Pi%`Tg7%gtyGm0h%q6I+=QGOJZ zN;5&JlJL#3)`133C}mceDTV>d!CFClTb?b_@lEy)+!DYk?a5_$$-iepDOrJjn7T>~sbagKZxmUt-`jS!JhxU`~5C!GEB z+sbBQJX^R6-xKsZN^?J;p&KEPC64MFgeI&Aj%llI#C}L7flEJu?ZEA&K-O)_OYi~X zCioza$I^hne0LhKkg}er#c|_V8ZuGtRCbbP0UarCIw8mn#>1ow8YcKA5`29y@i&QN zDlVRib1-dQ#cA`K=*OyIndxnV>&Va?OD;mzGOo0Ah0I{5X0cP4cf7sLr6VEZN!1ha zOEeux?lU0Vvm>wN(>a3gxD&%{8>Z+DM`4Z#%)sfayv;P+iMV|wIAxFbkU=yRv9g7v zo+d26C~YPjX>)@70*c6WBtr-aq#f@ufu#_+F=A;()Xgv^IO&dw&%#SEvTv4Q zXr>V9O(h|@kbG(^EvK?3nTI;VaGC#v&}vy3yo_nPSH2whVxYWQc~*~&He#bYAGKGi zUAs5Fy!pk=^3$qXSH~La*v`Y3a)0@wYF|wrsmmh`c?6g?sv|l2r}@w3%fG4gl|QL0 zRfj9TtM$KMHUIMPdH?g}x_rMO->=E{UuxaEQ#E<8G=)o(!O|S8I!5Z+AkA@NZWEO3 zJd_6;GNc)i1~q@4@O2ClPymkUN>dEvO0eK?&2+K~o!}t>g>RLnJ)8Z38mx#J1)kaP zErF)xbZ>tIZvO%JvGkzypmL|KU2kaDL97k~scF}%!!I;+Go0Y^L(l6><3U&I@*yK^nc{+uw*23PpQz&J6U5*!y|zRHQXI<{V?BTA2- z)*r{A-2LkVF30hI7P%GV=uks5_%zW+b0~Csn&4uczL)zf>qlSW)96e#uWw|GIr13F zVaylI=Rl4_q9`8WtJ~od+o7K$AztIpBjKkt{ybJB@#eAM6V>Oa?t4U8eD7HBNl#4t L$+2*jP*?v8IV$>f diff --git a/tests/__pycache__/test_prober.cpython-311-pytest-9.0.3.pyc b/tests/__pycache__/test_prober.cpython-311-pytest-9.0.3.pyc deleted file mode 100644 index d51b7f647f64b9fe95017bf4827dff0ba6722bd7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17714 zcmeHOO>7iNmd>im`c1oRV*?EsZ1dAlm;sxAW(;c@=$_F^u)8ojYWAm8O_ZUa>~cqz zp_|RpnANOQgBZ1MU5D1n+5o={>aA%YH8+D>5U? z1s;B8XLd-ZU%hzo;$>uH=8N}Uy!hAN-jspk;r*4uT+%T9g9_~<>>}TmfqZT-gN2I5 z<&gLd&ql6kgp2Xn#O1_n@^TXN$j^++DP{s%EDG4eVt~CY4wz;M zz&@4)>}M&!0cHWNV?BU_EOgPx^gf|3oX>l!n6Dx#7 zDPG>FE^KI5?P=4UxPDc=^pAM&J_%J!j2y;sQ@fE-NtCy!g5f0eR+S`6R#HyVN!^GN zSH*HH``@+ed*hyRER4HF+`(~YJ!>4%O`2>qqUFtAAN3w`y@eXv?kzCNJ=#@No8aAd z;Y#W~=)#rKxmfQPrg)M)EY13y9{0KSuHm`%>XP)5`&<)vwn_Y3l@v4Z<@cX|mY}bF z|LpwXR@^T0d~Vjxgj(UkTqbmT-L5_JpH61%(Zbv)p@%Yd%Ph>zl%Odhs3CCZSvqIi zd0rMAN5BE%ktg)wWf8n4kNSTNIg`&ATsQS}}HPW<3P*|P>sN+^AA6YvI z73X7fXS0pFL}CHjXxB&+XsbXXK7EQ%;Bis%C7TPJ4!kEYAL znJwbPT#F*wF|$R>eRQAlxK4K{ju+Sp%BFA(U7rVR1vLW(`N&q#dh2lptO^!;#U2JM zwyr%4@S%sXEou>?yxad!qytYHQXST(v_cLt4+b%g&|B`Rjn z)cMsL@Yk4mK&5XyN_4y{^NzAR74?o3kFDIK&&N5`s$*D1^ zF3QQUhwK`{vpM_lq+%Rg3azyci5e@Y5?CA;K7Ykw<0e*k}uhWnQ{ zV1=^qmk~YwMG>yoW{H}@5*0RUhb8)|cHYZdqTTGgpd}jrKeI&RUG&$>S)vJLi6&X< zja#DD`P+$Gh4QtwDVm}}R;m?JdM{^}iuNzIO9LwO!E=mG*`rrgq=QP43VFg(WA7S{rr3_35N-IvE1 z(*txY`9(wo{t_&O<;8>>_)D%K@E2IB)8)38L=p6h;y;chS6VvjInQZ`pU_eU0)Gj% zz{N7pReDB-v2NN$HcUSVKMPIqzXUN8zCA#pL0VIr62U#Jib)a>mvodCwID4M;WT=j z+(8QP9o3e;C@Of2M6PaW^<2%Dv+XYVyjBs?3xxSpgfEi{M)>}UItF8 zqnX;VcxJ^)*UinXSiq=8!m~6!a8gO|{1tkUTr3lG`KKffJ=1!#3SEY|ate8vyzU~zh7G{wz&HIr2 zkgg$)wdZ5}E!tEflu>|H6sfkLU_pyRWp&HVAcci>0imt}xf$$Ly~BWl2`Hkj3JZ#; zn|BygL3*SI5A-^GdtH6*<5WAKyT)kF)fK-N)l*%AdBCxhHdQv7UP)bDTRwgW0 zWh}hcoJ>%;DigZaMLYMCN)!oJS!5m{<-(_-1Ik%ngXz@Je@9pQdw^DWenx6(T9)Cv z3G4w_RWYwuidpML4*aC!?Vr3zIY)kZrfyC&%!#Hs0c53b{a1bOKI(gSx!ma6*X-MO z=ZA}@7eB6UU+EuQyilE7irhWEc!B=@>B7&%FX2RAyl_WQh%Pb7Q=RnaZCutYMcRCK zlq8ZuwNg6Ow|V4M|3JDyo#|BGYPp*d>fVI%X6Bbs-R4`n`mRKf2p01tf|)F?Eru7h zPua6Nk(34%o}^{-pyJV5(9|W4V0v8_uHbTS%2l>;X0aY$kjn*uMSUrBgsH|>`Qs-nru(I27KHuT)}kI zcU+&^Vw>1z$8vpYElpRG*RLj5qc9CSB{#Bi_zDY zBKQtdgBh%j>$y0AwU)_Wp82;(-JEKeQ%!RUh{)$0eq;^TE;X#}O=~;lbM`I8@13lh zqqVaD)sy(KMuExmu{qjoB?Y`nw&eG>Qp^&e(M;Ss zg_ogr5ukdicJa~k-%W6UQNBs;yLrlAs$KNIWxxY3nxy=W?qb9YmxpjNaMts0_QGEs1hnB67=$UpOn%NTY~5^(=T zfJ+LRstZbff2%-~&SIzQMAI8h6yN?QCW_bZI_Vi2M-azyBC7}OSsn$i)B{7RXJeh+45s3fw)e ztLY!HG>WKUsh~3sq_t3)$xgu0>cX|whz&)t-9wY)#eMH zl)M$15!*Kc^DQSMP|UNdhF~@ZCe&Fw_zx_c+5XMV#P4-sV(wU!++#7jn zgrXz0(<_POXYsr7rQSwjTQjk(p4j#@f(lV9z<&|ne-rR%t6|@L_uXpbZ)gujmpOZ^ zcCK!YK{cO4&K59vJ~qdiZCp|WhY7tw6|$O?w^yp2lO@RxTnY12&{D~k{Qg#fCbDQ# zG~iboP7g+YGiaXP82`=Y5TKUOs#M8&e2A|>mEeNyV-1@j!@8oVMD)#&A^?x*q_nLSArCk6-W0L2uWkjaE<*L(xj_i1jh;C z@Kt*eeCL#VS3H<@9LL0Cg2)Y53m{iV_YGckunt`lz0vW9hx>!WHm7**Z5$*vkgl zkkjjqNpcNivX*Jd!kExKN7;IB90Fq!|I-{3;nQz8e>-+_k`fc~dysygKvfz1Jb^0! zG&_sKMTVw1IpZ03?JR|ZSzv{%M;)ycE(N5fW}=k0}J+0K}9Wrn^| zxnD*&{+Fmo%jcZ>(Ms9n94}|fh1onW5S6;&Sqq#tMP||glR@sI>60!o?Ik4_U0&X2 zJ3N0mC++#|SihOX|G`!i76?{^Xg9rY)m^yi!fB_~cAu$1R>ms3Hl^1tQ;&jOyp1Ll z^{{`7yNBzZwC-*EZ20c*($(50jpUwYa?j#>s}r#;^S$MPy1Aoa?r54jfY46O0BI%N zzqF(JVQpRQVl7$k*}iygB|Y?2dgr6`P9$-s$D8T##Sgx=($!l(pSp*5z;F#q{lnF% z8n!%oTPoEaO0{W{I8ac#DJ6$0qDoR|$D_Lsw1Xgpb!%C@6=>-rDDhPUGDy4>Qp4s* zZ__Doc?AU}1-z0Sbjj~;rI;nMVy#=6y!ZX3V$;G1?_EY>1G2q#EZdJ>kj4)dmDcKJ z!CQDzz8rewCAfL_l943)&X>rc|D)=xyjS5yW`P0qv}7rOA@F_g^W^%@8K^0!L8H5L=zarZuszL!>8*ygyD^Fg!t9IT`QaU zq>eoaH&ssO)0{WlOnD$bzO27UxfJ)by0#;G_rRHZiMn1-?CN_) z-SGbACnZ$d!4dh#*6YTmZgb>j;s7qxbNCPY3?JqcpB-r*pZ7PKHx?T^B`oUN0j%e_ zToa4q@XYw99aqt9BQ|1$)Z-(511b?m+2#}D5v*#@w3BwKZkBW|CH-?^1)3HcNuG5* z86PPw{SJKmg}$f5zX871cK8jCWA^C%G+w8^_vvcvoYni5xSBnBKhyWV@SLWd+53ZH zj2^u|(D%OZ+$(s0NQ}{=_p^QP3(viR_lLz8J$m2jdtZ3&tlqaZ2-gN7QhS*ydDE4u zpVJ>*n_H+oyq5E_Uxv)|&1vMW=DfwtO$^?$d((vG`s+C_&}%s(FaU$(8j|aehKru( zFrQ;q0Rzg_ZyCPE!@!Q=!D{)GMR^o5dfn`(6e7x2)ry9#Z_SfKGO#w&k$ed~ko-`a`P92I4LP!xn zq}gARx(-DIS|bo48q3lI5Wv%nl9C3n1SbqEPqmU@V!rn$ccW>}G(G{ce*+HI3~?p_3uJ#44%Q@a)Q-%_mf%Ul5$5Ef=pcj?0YniF4P(7ruQ#jRxKd5iE1i(-Q@KaKz7OaWey8AvW{{9OO5J0yxyXYE&293 z3e;OOQ~q4E~1$-%uNJ;U#vNc!&gP8#pB{=E~sKJr?%-`lU@JMcfd zcX)1-x%*5H5+x5(h;0nemqlI!OqkbvL#iX9;Ot&3>%l?R#2~69f3SZM6Bbo^GNXf%f^tr ziR1#3w~&k@xrpQkK%S-fDY7Y4%m7hi{#8}DuEzZDQVK)az|xY3cO7K_(`*(;`4Nxo zh2smm7}S?4JI;ph*ab?G*mVHn3ccUMHqsH;LwLBbf1_nxg5y*DFBv#AtxHccO=|&J z-j2+Lmf%ArhnNe8qK6<-1Qcm-PY#Vc6cGqQMFfTOSeM#tq$n+(>QYBWiXsAKBnWiC z163Poz&@N;oQ`Ux;w6v4S>JEH`;8pP=HX`S9vXcHcW6H}ifuGe5Y-Qj;J$&d^30(T z-`>)9*#e9LTZi4-?nbq*Gwp2bCom&hL~;e^wrDH4go>WFlB;0Rxd}}2O#s)|eFInp zE8K*<58#*4jtu}(=-K_1$2Z~7SbuT{rJIdAO$&0fupODREy0tLBTUGh!VW@65kRED zr+*BhC?XK#(g+GoZnl|7Q5rPXrH*VAMFhGdL7)R3sM<&a_A~@xU=^yBif`ll53zoJ zr@U$K-^=gX9%7rlJ;X#Y{&haaCSKEHY?@+O{&PY12idzY6GqV8A7oIXp5I@4d>amp z;*)KZZa21@);w_3j?DR%;7P*~=KP`PAcPbFL>hcv^AcPbFL>k;~L)#8T z1cG035ELSAwV6m!8ZiaCbs^qP2 z^%*&jhbXy=K}NjF!3(az#R9$ju}HOHcp+nvIlpBkA712;svsTec)2zhm2XTe%GV+2 zm6A82p0I9e!mS@DYWl`9d@DIPAE8C@mEq54!En@p2|2@xtvUz=;0&ue!AOMV-$8~i zqVo4Jc+Hk!-p(-Q<=@0-8utUQ`@YQlyj-<+UFR0N1O52IvxmPpo*25WA7hf={jaGl zwzXp|*NnGgZJ`-&$J#s1cRK34QKFlO&ta RFQXWetf9{%xSepP{2MJt(MY~=b(b7|?TV6Z$*x@^5)&z?Da*3HWw)~I)OHR zEOiM8MGmUrHoDn|px{Zd;rxQ0iv9x9pb=q#o(cr;r4SIH2c7!f%l+({V(pk~AX>P8%3pOFBVHj)7Q4fVE?O})ZtZe>+> zpp;)XbGB*Oxep44Sy}NJK!OxJQFy)_12!KkHsZW%P4!FlU=JUuII5xULCbT|T8lVJ z9Y^c9oQM%Iq6XO`Ko1YU)E@KIqE2*8P}N98aflPy1Ad+ho@y~CW_@AAjJU1)<0;Bo z+%NAz;kjQ?K2WSP_JF7=Ye~NxT(LX@m3!b$v8#8FvI^V;XAymzs62KLDk4)Ei6gFU zNE%P9Excz`rfYK-+FMvhLK?A684Gck{3NyodtQs%C;k0V6sfOxJ5F5gE7yJtAHV~( z7KMk@V!=IbpF?&o`uur%oeLvXOE@tnvCd9VE$Jk8RqM3GQ`#TH(46Ew)zFdtUPL~f z-<~Y^YUC61KRj8|`)Z%RgnVj@zE9%p{H2Vv(eI=RDywNnIE|V-l3WX(MmUi%c#^e* zp@3i=xV4qKTgD2*L5AWtJ%cOSZ-E-B=#jK_M#w%p5Fw~rn zf`W&d@ljClP*YiJvYNN=Yqd%9LBX;ut#%H_na^5Tt>#85mP@exSVKHAecii zk6@9aT+XV8>T*`yTK!LPvufH_Vat@*$t<&#O#{=}I7Q7z@vEo!sZ#vbXg>g8cyoGc zYap;a)39KMp2_yp1QOy2bIDqzP>8$}~W%__~U7Atr2#auaGGTl_p%HJ~)?e@RNUQ?k`zDa3? zx`VlU6Zo)2Ab4G!3 z{&nP#FLT-rRAz&f1zzEkUJABc4puJl^>VW5X{k1A;WlU5*Wvk#v{JDzmTaq7D6gCJ z0`yzIi2GtOnDaTFtZur4Y`^4Prdb#fOaaCE100`i<*$1B(c;HT50{=K>Svq!g_eF{ z`^G-$`<$G8LC!w8Q6H^aPp>!0REtbC$P^=Qkdd9mN6Ywcd1v{_SW_Qq=_62Wa0mr5 z@{)|S$VihxZ1OR`pAk4t5DR!1aT|`xM04DsIc%J}55CMCF3Xo9dO}q!k zgm?yI@b;EJ2Ft(NAY(gMcdkB}YwDSno`Gt88bCwO)TdwQV^}1>)!0ii)?#Ij@kw|i zZ{Z|BD+oCTx5mSDAfP|T;a(w;SGyYOBC4VTa4e4+KnNmOMsV2rwCKYb2N=Aem3QS_ za2jAL*egiiwfrR>Ms4KByTIa&enw;pRRQ~0 z*tI?-{&yl^A;KC0wlqPTcO7N8!ff5YbhiZ!u!F&Zi6eD~1tVK=LVI`E#zWHBZEYK} zN5Os+)*RT{%*A2chv?9LG?J1AfAhIRHfhhfLv%2A(nx(0;Z~`kfo&J}tb&C79psi@ z(n^F)>4fZyPOG36V^%=lt*zvT1^Yf40f&johK+?^JX2i_vAO()QK&pq47S}hP!s3;A3tytI zZ0I?p%`g=bWFxn3npF#wC>Sk;@+$p4km2@POJE>@KGFIfeedImhZDOW*0rW~zNMYt zz8*xTn%a0v8wVsA+`V4E(ID9-$+k!q3j67kpQqn>k$$H>`!w22Pqfk#+wbq|$9Hc( zywV`2`TdfdZuw=7*;&}V4W*{vz$ru(nmA@>LEt(N!LbFE=(&4a;CRmfg7*XnB%;g< z90NUgCr$&*i@}2QbR2e3l79F)nDqMyWXd;C`bFPJ`bED{(k~P*=qdd|bpL(n*Ih@V zhFTL|bu9@In`AiVQA;@~>kFgLNFPx%oRMU-&`LoLWcdsco38q)cMsUFboV|^DALh` zDk*dDm9<{zeZI+i@4;az#9_Y|vC)p84Ugg@bKf}S`^GxzKTjXh36N0l(+dFUMFi6b zn5J38(j0<$1k6*!*d5n%_^yKjz-D$?7iM)FxZnPcJvge2dIC$o#jG6a?#QBx(6;C`sQmYJ?e3 zjJAB1+JmG29FIS{AXN;pk7BG8uMt(j45};#>q0*IJ@9|{`-f;pUo8$YH@?ouAhVXR z9!nBI$QUCHIW!frFhJemiwy)bGHb$0mw96OrtT6WBo9OKOUTBs)eiD&K;pJd8J1ti zyqHnaIBuM9l3vUxW#c#MUD8M*&+CLNY*`n2nWTg#$p&xHWf+ZK1@HL1dWH(e_|Si{7-4+u1Qyc4zA=i%=Ronw&%DXa9U!C znOTT8MPYy}k27I*Xm%?YXM(H}%hP0!b&E66>oB-`Oe7U1t87;KBSiN>hNR8h5e@UK z*3~A*0qKu%)Iic?rXwmf7YtD?Sg2b4s8l8sj7te&n`1#aX&Y}tuXo|Ge*W)>-}>7k zgbjr*GWgFB>p#^bH(LZu`kVV?q9Nb=+R2^szq|10!spu93vH}^yQyVcT6X*T{_&v( z8QPiM9ej8h3jBUahFX4^V~_w!O}~Lth$=L3%+9R9bsz-M?w}yzV*(7`(8_5AADlke zD@YGzE_GpR#U-Lu{{ZY`h=i|+rkc8{)FL3G)bKABYEft*PShcRSWv^psF!!G;rGBU zLv=6`4v}<+bjTkmHsAhjNf-GO99vkPjv{~6Ab)~%Lh6R;K>kE~P^$8XJ*Yy_?H*L2 zs6h{^kmYF*|F;{cLjqB9A=%)U9KluM8)VnNj*^4zrot0}2Y=BKs}W|$C2p}e07$5x zp@drS^WIQ+eJb2Ajg4xtumag&FY^qcaY%&w5%tR;z%Z(vjkP1`EU198ZUMn%Iq^L0 zC7!3VDOV4Se$>!{A4Si=q|EjUimZ8xk|l^)l=8MyQf;P7T>TU?76W|c%u?0fq`!u7 zF_L1PV^V8hA7q);Y&`yiW{n8Jo3rs@?5~ztl zJ0}rUXyTZiO9Iz{5JW7EiV=HG0{pNEc@3rtR_jy@$ma(byrGqk>-F||4E74rcP%qn zlpS!B{*MNjsqdc2@m~x~@`MZM-$`nINDDUnOhBOAgoon?1C1t^S@PE8N~L@cUlYs4 zu)H}-cCZi-MeClTj$yCtnZkHry6G^vRH_&oMRS=N(2gR~`XhjYsH&>_7&Pkfm)d53 z`^tDjz7OI=efK~Spg*RbIZy;R7FS0O6ah}euqwc@gnIHo5nxDF#}AZF!1(_srugjU From 8c5198fae641745b6514babbf2e1bfa89b7cbf58 Mon Sep 17 00:00:00 2001 From: Boris Rybalkin Date: Wed, 20 May 2026 17:46:16 +0000 Subject: [PATCH 65/65] Retry slide composite chunk on NVENC OOM-kill with libx264 fallback When the kernel OOM killer reaped ffmpeg mid slide-composite chunk (exit -9 / 137), the whole video failed. NVENC's pinned host buffers push long filter_complex graphs over the memory budget on some videos. Catch SIGKILL only (not generic ffmpeg errors), re-run that chunk with libx264, and stay on CPU for the remaining chunks of the same video so the next chunk does not pay the kill cost again. Non-OOM failures still propagate. Adds get_video_encoder_cpu() and a tests/test_slides.py spy for the retry path. --- mtslinker/ffmpeg.py | 7 ++ mtslinker/slides.py | 63 ++++++++++++--- tests/test_ffmpeg.py | 11 +++ tests/test_slides.py | 178 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 248 insertions(+), 11 deletions(-) create mode 100644 tests/test_slides.py diff --git a/mtslinker/ffmpeg.py b/mtslinker/ffmpeg.py index 7afb154..4f1d1d7 100644 --- a/mtslinker/ffmpeg.py +++ b/mtslinker/ffmpeg.py @@ -66,6 +66,13 @@ def get_video_encoder_fast(self) -> list: return ['-c:v', 'h264_nvenc', '-preset', 'p1', '-cq', '28'] return ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23'] + def get_video_encoder_cpu(self) -> list: + # Fallback for callers that need to bypass NVENC after a system + # OOM-kill — NVENC's pinned host buffers can push long filter_complex + # jobs over the memory budget; libx264 fits in less RAM at the cost + # of CPU time. + return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] + def run(self, cmd: list, description: str = 'ffmpeg'): logging.debug(f'Running {description}: {" ".join(cmd[:6])}...') result = subprocess.run(cmd, capture_output=True, text=True) diff --git a/mtslinker/slides.py b/mtslinker/slides.py index 772bd22..04b51bb 100644 --- a/mtslinker/slides.py +++ b/mtslinker/slides.py @@ -1,6 +1,7 @@ import logging import os import shutil +import subprocess from typing import Dict, List from mtslinker.ffmpeg import FFmpegRunner @@ -14,6 +15,10 @@ class SlideCompositor: SLIDE_W = 960 CAM_W = 320 CHUNK_SECS = 1800 + # Exit codes that mean the kernel killed ffmpeg for memory. Python's + # subprocess reports SIGKILL as -9; some shells/wrappers surface it as + # 128+9 = 137. Both indicate the same OOM-kill scenario. + _OOM_KILL_CODES = (-9, 137) def __init__(self, ffmpeg: FFmpegRunner): self.ffmpeg = ffmpeg @@ -35,7 +40,6 @@ def composite(self, video_path: str, slide_events: List[Dict], ) n_chunks = max(1, int(total_duration + self.CHUNK_SECS - 1) // self.CHUNK_SECS) - enc_args = self.ffmpeg.get_video_encoder() if n_chunks == 1: cmd = [ @@ -43,7 +47,8 @@ def composite(self, video_path: str, slide_events: List[Dict], '-i', video_path, '-i', slide_track_path, '-filter_complex', filter_graph, '-map', '[out]', '-map', '0:a?', - *enc_args, '-c:a', 'copy', '-r', '25', '-shortest', + *self.ffmpeg.get_video_encoder(), + '-c:a', 'copy', '-r', '25', '-shortest', output_path, ] self.ffmpeg.run(cmd, description='composite slides + webcam') @@ -52,23 +57,59 @@ def composite(self, video_path: str, slide_events: List[Dict], chunks_dir = os.path.join(tmp_dir, 'composite_chunks') os.makedirs(chunks_dir, exist_ok=True) chunk_paths = [] - - for ci in range(n_chunks): - ss = ci * self.CHUNK_SECS - chunk_path = os.path.join(chunks_dir, f'chunk_{ci:03d}.mp4') - cmd = [ + # NVENC keeps pinned host buffers per stream; on a long, heavy + # filter_complex graph the OOM-killer can SIGKILL ffmpeg mid + # chunk. Once that has happened, the NVENC budget for *this* + # video is over budget — stay on CPU for the rest of the chunks + # instead of paying the kill cost again on each one. + use_cpu_encoder = False + + def build_chunk_cmd(ss_arg, chunk_path_arg, enc_args): + return [ 'ffmpeg', '-y', '-v', 'warning', - '-ss', str(ss), '-t', str(self.CHUNK_SECS), + '-ss', str(ss_arg), '-t', str(self.CHUNK_SECS), '-i', video_path, - '-ss', str(ss), '-t', str(self.CHUNK_SECS), + '-ss', str(ss_arg), '-t', str(self.CHUNK_SECS), '-i', slide_track_path, '-filter_complex', filter_graph, '-map', '[out]', '-map', '0:a?', *enc_args, '-c:a', 'aac', '-b:a', '192k', '-r', '25', '-shortest', - chunk_path, + chunk_path_arg, ] - self.ffmpeg.run(cmd, description=f'composite chunk {ci+1}/{n_chunks}') + + for ci in range(n_chunks): + ss = ci * self.CHUNK_SECS + chunk_path = os.path.join(chunks_dir, f'chunk_{ci:03d}.mp4') + label = f'composite chunk {ci+1}/{n_chunks}' + + if use_cpu_encoder: + self.ffmpeg.run( + build_chunk_cmd(ss, chunk_path, + self.ffmpeg.get_video_encoder_cpu()), + description=f'{label} (cpu)', + ) + else: + try: + self.ffmpeg.run( + build_chunk_cmd(ss, chunk_path, + self.ffmpeg.get_video_encoder()), + description=label, + ) + except subprocess.CalledProcessError as e: + if e.returncode not in self._OOM_KILL_CODES: + raise + logging.warning( + f'{label} OOM-killed (exit {e.returncode}); ' + f'retrying this chunk with CPU encoder and ' + f'falling back to CPU for any remaining chunks' + ) + use_cpu_encoder = True + self.ffmpeg.run( + build_chunk_cmd(ss, chunk_path, + self.ffmpeg.get_video_encoder_cpu()), + description=f'{label} (cpu retry)', + ) chunk_paths.append(chunk_path) logging.info(f'Composite chunk {ci+1}/{n_chunks} done') diff --git a/tests/test_ffmpeg.py b/tests/test_ffmpeg.py index bdb35e2..da827e4 100644 --- a/tests/test_ffmpeg.py +++ b/tests/test_ffmpeg.py @@ -27,6 +27,17 @@ def test_get_video_encoder_fast_returns_codec_flag(ffmpeg): assert '-c:v' in enc +def test_get_video_encoder_cpu_always_returns_libx264(ffmpeg): + """The CPU fallback must never pick NVENC, even when NVENC is available. + + It exists specifically to bypass a system OOM-kill of NVENC, so it + cannot conditionally fall back to the very thing it is meant to + replace. + """ + enc = ffmpeg.get_video_encoder_cpu() + assert enc[enc.index('-c:v') + 1] == 'libx264' + + def test_run_success(ffmpeg): result = ffmpeg.run(['ffmpeg', '-version'], description='version') assert result.returncode == 0 diff --git a/tests/test_slides.py b/tests/test_slides.py new file mode 100644 index 0000000..8b0b652 --- /dev/null +++ b/tests/test_slides.py @@ -0,0 +1,178 @@ +import os +import subprocess + +import pytest + +from mtslinker.slides import SlideCompositor + + +class _FakeFFmpeg: + """Composition substitute for FFmpegRunner used in SlideCompositor tests. + + Records every (cmd, description) call. To make the next call matching + a description substring fail with a specific returncode, register it + via fail_with() before invoking the code under test. + """ + + def __init__(self): + self.calls = [] + self._pending_failures = [] + self.nvenc_available = True + self.cuda_overlay_available = False + + def get_video_encoder(self): + return ['-c:v', 'h264_nvenc', '-preset', 'p4', '-cq', '23'] + + def get_video_encoder_fast(self): + return ['-c:v', 'h264_nvenc', '-preset', 'p1', '-cq', '28'] + + def get_video_encoder_cpu(self): + return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] + + def fail_with(self, description_substring, returncode): + self._pending_failures.append((description_substring, returncode)) + + def run(self, cmd, description='ffmpeg'): + self.calls.append((list(cmd), description)) + for i, (sub, rc) in enumerate(self._pending_failures): + if sub == description: + self._pending_failures.pop(i) + raise subprocess.CalledProcessError(rc, cmd[0], '', '') + out = cmd[-1] + if isinstance(out, str) and not out.startswith('-'): + try: + with open(out, 'wb') as f: + f.write(b'\x00') + except OSError: + pass + + +class _StubSlideTrackCompositor(SlideCompositor): + """SlideCompositor with the slide-track build stubbed out. + + The OOM-retry logic under test lives in composite()'s chunked branch; + _build_slide_track does many small ffmpeg calls that would clutter the + recorded call list and tell us nothing about retry behavior. Touching + a single empty file is enough — the chunked composite reads the path, + not the contents. + """ + + def _build_slide_track(self, slide_events, tmp_dir, total_duration): + path = os.path.join(tmp_dir, 'slide_track.mp4') + open(path, 'wb').close() + return path + + +def _enc_codec(cmd): + return cmd[cmd.index('-c:v') + 1] + + +def _composite_chunk_calls(calls): + # Match the per-chunk encode calls only; the final 'concat composite + # chunks' concat-demuxer step also contains "composite chunk" as a + # substring and would otherwise pollute the assertions. + return [(c, d) for c, d in calls if d.startswith('composite chunk')] + + +def _slide_events(): + return [{'time': 0, 'local_path': 'x.jpg', 'slide_number': 1}] + + +def test_composite_chunk_retries_on_oom_kill_with_cpu_encoder(tmp_path): + """exit -9 SIGKILL on NVENC must trigger a CPU retry of that chunk. + + Reproduces the production OOM-kill on `composite chunk 2/8` (URL + ...record-new/2413910825): the slide compositor's NVENC pipeline was + killed by the kernel OOM killer mid-chunk. The correct response is to + retry that chunk with libx264 and stay on CPU for the rest of the + video so the next chunk does not pay the kill cost again. + """ + ffmpeg = _FakeFFmpeg() + ffmpeg.fail_with('composite chunk 2/3', -9) + compositor = _StubSlideTrackCompositor(ffmpeg) + + video = os.path.join(str(tmp_path), 'video.mp4') + open(video, 'wb').close() + out = os.path.join(str(tmp_path), 'final.mp4') + compositor.composite(video, _slide_events(), out, str(tmp_path), 5400.0) + + chunks = _composite_chunk_calls(ffmpeg.calls) + descs = [d for _, d in chunks] + assert descs == [ + 'composite chunk 1/3', + 'composite chunk 2/3', + 'composite chunk 2/3 (cpu retry)', + 'composite chunk 3/3 (cpu)', + ], descs + assert _enc_codec(chunks[0][0]) == 'h264_nvenc' + assert _enc_codec(chunks[1][0]) == 'h264_nvenc' + assert _enc_codec(chunks[2][0]) == 'libx264' + assert _enc_codec(chunks[3][0]) == 'libx264' + + +def test_composite_chunk_does_not_retry_on_non_oom_failure(tmp_path): + """A regular nonzero exit must propagate — only SIGKILL means OOM. + + Falling back to CPU on every ffmpeg error would hide real bugs (bad + inputs, codec errors, etc.). Only the kernel OOM killer signals + memory pressure, so only that should trigger the silent retry. + """ + ffmpeg = _FakeFFmpeg() + ffmpeg.fail_with('composite chunk 1/3', 1) + compositor = _StubSlideTrackCompositor(ffmpeg) + + video = os.path.join(str(tmp_path), 'video.mp4') + open(video, 'wb').close() + out = os.path.join(str(tmp_path), 'final.mp4') + with pytest.raises(subprocess.CalledProcessError): + compositor.composite(video, _slide_events(), out, str(tmp_path), 5400.0) + + chunks = _composite_chunk_calls(ffmpeg.calls) + assert len(chunks) == 1 + assert chunks[0][1] == 'composite chunk 1/3' + + +def test_composite_chunk_treats_exit_137_as_oom_kill(tmp_path): + """Some shells surface SIGKILL as 128+9=137 instead of Python's -9. + + Both must trigger the same CPU retry path so the fallback is not + sensitive to the kernel/wrapper that reaped ffmpeg. + """ + ffmpeg = _FakeFFmpeg() + ffmpeg.fail_with('composite chunk 1/3', 137) + compositor = _StubSlideTrackCompositor(ffmpeg) + + video = os.path.join(str(tmp_path), 'video.mp4') + open(video, 'wb').close() + out = os.path.join(str(tmp_path), 'final.mp4') + compositor.composite(video, _slide_events(), out, str(tmp_path), 5400.0) + + chunks = _composite_chunk_calls(ffmpeg.calls) + descs = [d for _, d in chunks] + assert descs == [ + 'composite chunk 1/3', + 'composite chunk 1/3 (cpu retry)', + 'composite chunk 2/3 (cpu)', + 'composite chunk 3/3 (cpu)', + ], descs + + +def test_composite_chunk_uses_nvenc_when_nothing_fails(tmp_path): + """The happy path must keep using NVENC — the CPU fallback is only + engaged after an actual OOM-kill, never preemptively.""" + ffmpeg = _FakeFFmpeg() + compositor = _StubSlideTrackCompositor(ffmpeg) + + video = os.path.join(str(tmp_path), 'video.mp4') + open(video, 'wb').close() + out = os.path.join(str(tmp_path), 'final.mp4') + compositor.composite(video, _slide_events(), out, str(tmp_path), 5400.0) + + chunks = _composite_chunk_calls(ffmpeg.calls) + assert [d for _, d in chunks] == [ + 'composite chunk 1/3', + 'composite chunk 2/3', + 'composite chunk 3/3', + ] + for cmd, _ in chunks: + assert _enc_codec(cmd) == 'h264_nvenc'