diff --git a/.gitignore b/.gitignore index ccb6a9f..88c9691 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ -*/ *.log *.mp4 *.mp3 -!mtslinker !get_sessionId.mp4 -!setup.py \ No newline at end of file +__pycache__/ +*.pyc +*.egg-info/ diff --git a/Dockerfile b/Dockerfile index 1c0ad66..e19049c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,22 @@ FROM python:3.13-slim WORKDIR /app -# Копируем только requirements.txt сначала для кэширования +# Install ffmpeg +RUN apt-get update && \ + apt-get install -y --no-install-recommends ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +# Copy requirements first for caching COPY requirements.txt . -# Устанавливаем зависимости +# Install dependencies RUN pip install --no-cache-dir --upgrade pip && \ pip install --no-cache-dir -r requirements.txt -# Копируем остальные файлы проекта +# Copy project files COPY . . -# Устанавливаем пакет (если он setup как пакет) +# Install package RUN if [ -f "setup.py" ]; then pip install --no-cache-dir .; fi ENTRYPOINT ["mtslinker"] - diff --git a/mtslinker/__init__.py b/mtslinker/__init__.py index 6f621ca..1518e9f 100644 --- a/mtslinker/__init__.py +++ b/mtslinker/__init__.py @@ -1,3 +1,3 @@ from mtslinker.utils import initialize_logger -initialize_logger() \ No newline at end of file +initialize_logger() diff --git a/mtslinker/audio.py b/mtslinker/audio.py new file mode 100644 index 0000000..6d2c1ab --- /dev/null +++ b/mtslinker/audio.py @@ -0,0 +1,175 @@ +import logging +import os +import shutil +import subprocess +from typing import List, Union + +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.timeline import AudioTrack + + +class AudioMerger: + """Merges multiple audio tracks onto a video using batched amix.""" + + BATCH_SIZE = 8 + + def __init__(self, ffmpeg: FFmpegRunner, prober: MediaProber): + self.ffmpeg = ffmpeg + self.prober = prober + + def merge(self, video_path: str, + audio_files: List[Union[AudioTrack, tuple]], + tmp_dir: str, output_path: str, + total_duration: float = 0) -> str: + video_duration = total_duration or self.prober.get_duration(video_path) + + # Normalize to AudioTrack objects + tracks = [] + for af in audio_files: + if isinstance(af, AudioTrack): + tracks.append(af) + else: + tracks.append(AudioTrack(path=af[0], start_time=af[1])) + + # Step 1: Mix in batches with inline adelay + batch_outputs = [] + total_batches = (len(tracks) + self.BATCH_SIZE - 1) // self.BATCH_SIZE + for batch_idx, batch_start in enumerate( + range(0, len(tracks), self.BATCH_SIZE) + ): + batch = tracks[batch_start:batch_start + self.BATCH_SIZE] + batch_out = os.path.join(tmp_dir, f'audio_batch_{batch_idx}.m4a') + + inputs = [] + filter_parts = [] + mix_labels = [] + for j, track in enumerate(batch): + inputs.extend(['-i', track.path]) + delay_ms = int(track.start_time * 1000) + label = f'a{j}' + filter_parts.append( + f'[{j}:a]adelay={delay_ms}|{delay_ms},' + f'apad=whole_dur={video_duration},' + f'atrim=0:{video_duration},' + f'asetpts=PTS-STARTPTS[{label}]' + ) + mix_labels.append(f'[{label}]') + + if len(batch) == 1: + filter_graph = filter_parts[0].rsplit('[', 1)[0] + else: + filter_graph = ( + ';'.join(filter_parts) + ';' + + ''.join(mix_labels) + + f'amix=inputs={len(batch)}:duration=longest:normalize=0' + ) + + try: + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-filter_complex', filter_graph, + '-c:a', 'aac', '-b:a', '128k', + '-ar', '44100', '-ac', '2', + batch_out, + ], + description=f'amix batch {batch_idx+1} ' + f'({len(batch)} tracks, offset {batch[0].start_time:.0f}-' + f'{batch[-1].start_time:.0f}s)', + ) + batch_outputs.append(batch_out) + except subprocess.CalledProcessError: + logging.warning( + f'Audio batch {batch_idx+1} failed, skipping {len(batch)} tracks' + ) + logging.info(f'Audio batch {batch_idx+1}/{total_batches} done') + + if not batch_outputs: + logging.warning('All audio batches failed, skipping audio overlay') + shutil.move(video_path, output_path) + return output_path + + # Step 2: Tree-reduce batch outputs + round_num = 0 + current_paths = batch_outputs + while len(current_paths) > 1: + round_num += 1 + next_paths = [] + for batch_start in range(0, len(current_paths), self.BATCH_SIZE): + batch = current_paths[batch_start:batch_start + self.BATCH_SIZE] + if len(batch) == 1: + next_paths.append(batch[0]) + continue + + batch_out = os.path.join( + tmp_dir, f'audio_reduce_r{round_num}_b{batch_start}.m4a' + ) + inputs = [] + for bp in batch: + inputs.extend(['-i', bp]) + + labels = ''.join(f'[{j}:a]' for j in range(len(batch))) + amix_filter = ( + f'{labels}amix=inputs={len(batch)}' + f':duration=longest:normalize=0' + ) + + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-filter_complex', amix_filter, + '-c:a', 'aac', '-b:a', '128k', + '-ar', '44100', '-ac', '2', + batch_out, + ], + description=f'amix reduce round {round_num}, {len(batch)} tracks', + ) + next_paths.append(batch_out) + + for bp in batch: + try: + os.remove(bp) + except OSError: + pass + + logging.info( + f'Audio reduce round {round_num}: {len(current_paths)} -> ' + f'{len(next_paths)} tracks' + ) + current_paths = next_paths + + mixed_audio_path = current_paths[0] + + # Step 3: Overlay mixed audio onto video + logging.info('Overlaying mixed audio onto video...') + video_has_audio = self.prober.has_audio(video_path) + if video_has_audio: + # Mix video's audio with merged audio, normalize sample rates + filter_graph = ( + '[0:a]aresample=44100[va];' + '[va][1:a]amix=inputs=2:duration=first:normalize=0[aout]' + ) + audio_map = ['-map', '[aout]'] + else: + # Video has no audio, just use merged audio + audio_map = ['-map', '1:a'] + filter_graph = None + + cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', video_path, + '-i', mixed_audio_path, + ] + if filter_graph: + cmd.extend(['-filter_complex', filter_graph]) + cmd.extend([ + '-map', '0:v', *audio_map, + '-c:v', 'copy', + '-c:a', 'aac', '-b:a', '192k', + output_path, + ]) + self.ffmpeg.run(cmd, description='overlay mixed audio onto video') + return output_path diff --git a/mtslinker/downloader.py b/mtslinker/downloader.py index 99f886a..c2f178e 100644 --- a/mtslinker/downloader.py +++ b/mtslinker/downloader.py @@ -1,17 +1,56 @@ import os -from typing import Dict, Union +import subprocess +from typing import Dict, List, Tuple, Union +from concurrent.futures import ThreadPoolExecutor, as_completed import httpx import tqdm import logging TIMEOUT_SETTINGS = httpx.Timeout(None, connect=None) +CHUNK_SIZE = 1024 * 1024 # 1 MB +MAX_PARALLEL_DOWNLOADS = 4 + + +def _storage_to_hls_url(storage_url: str) -> str: + """Convert events-storage URL to events-delivery-records HLS URL.""" + return ( + storage_url + .replace('events-storage.webinar.ru/api-storage/files/wowza/', + 'events-delivery-records.webinar.ru/record/') + + '/playlist.m3u8' + ) + + +def _hls_is_available(hls_url: str) -> bool: + """Check if an HLS playlist is reachable and non-empty.""" + try: + with httpx.Client(timeout=httpx.Timeout(10)) as client: + r = client.get(hls_url, headers={'User-Agent': 'Mozilla/5.0'}) + return r.status_code == 200 and '#EXTM3U' in r.text + except Exception: + return False + + +def download_hls_chunk(hls_url: str, save_path: str) -> str: + """Download an HLS stream (video, audio, or both) to mp4 using ffmpeg.""" + if not os.path.exists(save_path): + subprocess.run( + [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', hls_url, + '-c', 'copy', + save_path, + ], + check=True, + ) + return save_path def construct_json_data_url(event_session_id: str, recording_id: str) -> str: if not event_session_id: raise ValueError('Missing webinar event session ID.') - + if not recording_id: return f'https://my.mts-link.ru/api/eventsessions/{event_session_id}/record?withoutCuts=false' return f'https://my.mts-link.ru/api/event-sessions/{event_session_id}/record-files/{recording_id}/flow?withoutCuts=false' @@ -30,7 +69,7 @@ def fetch_json_data(url: str, session_id: Union[str, None]) -> Dict: }, cookies=cookies ) - + try: error_data = response.json() if error_data.get("error", {}).get("code") == 403: @@ -41,24 +80,139 @@ def fetch_json_data(url: str, session_id: Union[str, None]) -> Dict: return except Exception: logging.warning('Server response does not contain JSON.') - + response.raise_for_status() return response.json() -def download_video_chunk(video_url: str, save_directory: str) -> str: +def _validate_downloaded_file(file_path: str) -> bool: + """Quick check that a downloaded media file is not corrupt.""" + result = subprocess.run( + ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', + '-of', 'csv=p=0', file_path], + capture_output=True, text=True, + ) + return result.returncode == 0 + + +def download_video_chunk(video_url: str, save_directory: str, max_retries: int = 2) -> str: filename = os.path.basename(video_url) file_path = os.path.join(save_directory, filename) - if not os.path.exists(file_path): - with open(file_path, 'wb') as file: - with httpx.Client(timeout=TIMEOUT_SETTINGS) as client: - with client.stream('GET', video_url) as response: - response.raise_for_status() - total_size = int(response.headers.get('content-length', 0)) - with tqdm.tqdm(total=total_size, unit='B', unit_scale=True, - desc=f'Downloading {filename}') as progress: - for chunk in response.iter_bytes(chunk_size=8192): - file.write(chunk) - progress.update(len(chunk)) + if os.path.exists(file_path): + if _validate_downloaded_file(file_path): + return file_path + logging.warning(f'Existing file corrupt, re-downloading: {filename}') + os.remove(file_path) + + for attempt in range(max_retries + 1): + hls_url = _storage_to_hls_url(video_url) + hls_ok = _hls_is_available(hls_url) + if hls_ok: + logging.info(f'Downloading via HLS: {filename}') + try: + download_hls_chunk(hls_url, file_path) + except subprocess.CalledProcessError: + logging.warning(f'HLS download failed for {filename}, falling back to direct') + if os.path.exists(file_path): + os.remove(file_path) + hls_ok = False + if not hls_ok: + logging.info(f'Downloading direct: {filename}') + with open(file_path, 'wb') as file: + with httpx.Client(timeout=TIMEOUT_SETTINGS) as client: + with client.stream('GET', video_url) as response: + response.raise_for_status() + total_size = int(response.headers.get('content-length', 0)) + with tqdm.tqdm(total=total_size, unit='B', unit_scale=True, + desc=f'Downloading {filename}') as progress: + for chunk in response.iter_bytes(chunk_size=CHUNK_SIZE): + file.write(chunk) + progress.update(len(chunk)) + + if _validate_downloaded_file(file_path): + return file_path + + if attempt < max_retries: + logging.warning(f'Downloaded file corrupt (attempt {attempt+1}), retrying: {filename}') + os.remove(file_path) + else: + logging.error(f'File still corrupt after {max_retries+1} attempts: {filename}') + return file_path + + +def download_slide_images( + slide_events: List[Dict], + save_directory: str, +) -> List[Dict]: + """Download slide images and add local_path to each event. + + Deduplicates by URL so each unique slide is downloaded once. + """ + slides_dir = os.path.join(save_directory, 'slides') + os.makedirs(slides_dir, exist_ok=True) + + url_to_path = {} + for se in slide_events: + url = se['slide_url'] + if url in url_to_path: + continue + local_path = os.path.join(slides_dir, f"slide_{se['slide_number']}.jpg") + if not os.path.exists(local_path): + try: + with httpx.Client(timeout=httpx.Timeout(30)) as client: + r = client.get(url) + r.raise_for_status() + with open(local_path, 'wb') as f: + f.write(r.content) + except Exception as e: + logging.warning(f"Failed to download slide {se['slide_number']}: {e}") + continue + url_to_path[url] = local_path + + result = [] + for se in slide_events: + path = url_to_path.get(se['slide_url']) + if path: + result.append({**se, 'local_path': path}) + + logging.info(f'Downloaded {len(url_to_path)} unique slide images') + return result + + +def download_chunks_parallel( + chunks: list, + save_directory: str, + max_workers: int = MAX_PARALLEL_DOWNLOADS, +) -> list: + """Download multiple chunks in parallel. + + Args: + chunks: List of (url, start_time[, conf_id[, is_admin]]) tuples. + save_directory: Directory to save files to. + max_workers: Maximum number of parallel downloads. + + Returns: + List of (file_path, start_time, conf_id, is_admin) tuples in original order. + """ + results = [None] * len(chunks) + + def _download(index, url, start_time, conf_id, is_admin): + path = download_video_chunk(url, save_directory) + return index, path, start_time, conf_id, is_admin + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for i, chunk in enumerate(chunks): + url, start_time = chunk[0], chunk[1] + conf_id = chunk[2] if len(chunk) > 2 else None + is_admin = chunk[3] if len(chunk) > 3 else False + futures.append( + executor.submit(_download, i, url, start_time, conf_id, is_admin) + ) + for future in as_completed(futures): + idx, path, start_time, conf_id, is_admin = future.result() + results[idx] = (path, start_time, conf_id, is_admin) + + return results diff --git a/mtslinker/ffmpeg.py b/mtslinker/ffmpeg.py new file mode 100644 index 0000000..4f1d1d7 --- /dev/null +++ b/mtslinker/ffmpeg.py @@ -0,0 +1,87 @@ +import logging +import shutil +import subprocess + + +class FFmpegRunner: + """Handles ffmpeg/ffprobe execution and GPU detection.""" + + def __init__(self): + self.nvenc_available = None + self.cuda_overlay_available = None + self._check_tools() + + def _check_tools(self): + for tool in ('ffmpeg', 'ffprobe'): + if not shutil.which(tool): + raise RuntimeError( + f'{tool} is not installed. ' + 'Install it with: apt install ffmpeg / brew install ffmpeg' + ) + + def detect_gpu(self): + if self.nvenc_available is not None: + return + self.nvenc_available = self._has_nvenc() + self.cuda_overlay_available = ( + self._has_cuda_overlay() if self.nvenc_available else False + ) + if self.cuda_overlay_available: + logging.info('CUDA overlay + NVENC detected, using full GPU pipeline') + elif self.nvenc_available: + logging.info('NVENC detected (no CUDA overlay), using GPU encoder only') + else: + logging.info('No GPU support, using CPU pipeline') + + def _has_nvenc(self) -> bool: + try: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-f', 'lavfi', '-i', + 'nullsrc=s=64x64:d=0.1', '-c:v', 'h264_nvenc', '-f', 'null', '-'], + capture_output=True, timeout=10, + ) + return result.returncode == 0 + except Exception: + return False + + def _has_cuda_overlay(self) -> bool: + try: + result = subprocess.run( + ['ffmpeg', '-v', 'error', '-filters'], + capture_output=True, text=True, timeout=10, + ) + return 'overlay_cuda' in result.stdout + except Exception: + return False + + def get_video_encoder(self) -> list: + self.detect_gpu() + if self.nvenc_available: + return ['-c:v', 'h264_nvenc', '-preset', 'p4', '-cq', '23'] + return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] + + def get_video_encoder_fast(self) -> list: + self.detect_gpu() + if self.nvenc_available: + return ['-c:v', 'h264_nvenc', '-preset', 'p1', '-cq', '28'] + return ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23'] + + def get_video_encoder_cpu(self) -> list: + # Fallback for callers that need to bypass NVENC after a system + # OOM-kill — NVENC's pinned host buffers can push long filter_complex + # jobs over the memory budget; libx264 fits in less RAM at the cost + # of CPU time. + return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] + + def run(self, cmd: list, description: str = 'ffmpeg'): + logging.debug(f'Running {description}: {" ".join(cmd[:6])}...') + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + stderr = result.stderr.strip() if result.stderr else '(no stderr)' + logging.error(f'{description} failed (exit {result.returncode}):\n{stderr}') + raise subprocess.CalledProcessError( + result.returncode, cmd[0], result.stdout, result.stderr + ) + if result.stderr and result.stderr.strip(): + logging.debug(f'{description} stderr: {result.stderr.strip()[:500]}') + return result diff --git a/mtslinker/grid.py b/mtslinker/grid.py new file mode 100644 index 0000000..21fdc3f --- /dev/null +++ b/mtslinker/grid.py @@ -0,0 +1,249 @@ +import math +from typing import List, Optional, Union + +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.timeline import GridSource + + +def _even(x: int) -> int: + """Round down to nearest even number.""" + return x & ~1 + + +def _build_audio_filter(sources: List[GridSource], duration: float): + """Build amix filter graph and map args for a list of sources. + + Returns (filter_str, audio_map) where filter_str starts with ';' + and audio_map is e.g. ['-map', '[aout]']. + """ + audio_labels = [] + afilter = '' + for i, src in enumerate(sources): + if not src.has_audio: + continue + alabel = f'a{i}' + afilter += ( + f';[{i}:a]apad=whole_dur={duration},' + f'atrim=0:{duration}[{alabel}]' + ) + audio_labels.append(f'[{alabel}]') + + if len(audio_labels) > 1: + afilter += ( + ';' + ''.join(audio_labels) + + f'amix=inputs={len(audio_labels)}:duration=longest:normalize=0[aout]' + ) + return afilter, ['-map', '[aout]'] + elif len(audio_labels) == 1: + afilter += f';{audio_labels[0]}acopy[aout]' + return afilter, ['-map', '[aout]'] + else: + return '', [] + + +class GridLayout: + """Composites multiple streams into an equal-size grid using xstack.""" + + def __init__(self, ffmpeg: FFmpegRunner): + self.ffmpeg = ffmpeg + + @staticmethod + def compute_layout(n: int): + cols = math.ceil(math.sqrt(n)) + rows = math.ceil(n / cols) + return cols, rows + + def composite(self, sources: List[GridSource], + duration: float, output_path: str, + target_w: int, target_h: int, + mix_all_audio: bool = False) -> str: + """Composite streams into a grid. Output is exactly target_w x target_h.""" + n = len(sources) + cols, rows = self.compute_layout(n) + cell_w = _even(target_w // cols) + cell_h = _even(target_h // rows) + + inputs = [] + filter_parts = [] + labels = [] + + for i, src in enumerate(sources): + inputs.extend(['-ss', str(src.offset), '-i', src.path]) + label = f'v{i}' + filter_parts.append( + f'[{i}:v]scale=w=min({cell_w}\\,iw):h=min({cell_h}\\,ih)' + f':force_original_aspect_ratio=decrease,' + f'pad={cell_w}:{cell_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[{label}]' + ) + labels.append(f'[{label}]') + + total_cells = cols * rows + for i in range(n, total_cells): + idx = len(sources) + i - n + inputs.extend(['-f', 'lavfi', '-i', + f'color=black:s={cell_w}x{cell_h}:d={duration}:r=25']) + labels.append(f'[{idx}:v]') + + layout_parts = [] + for i in range(total_cells): + c = i % cols + r = i // cols + layout_parts.append(f'{c * cell_w}_{r * cell_h}') + layout = '|'.join(layout_parts) + + filter_graph = ';'.join(filter_parts) + if filter_graph: + filter_graph += ';' + filter_graph += ( + ''.join(labels) + + f'xstack=inputs={total_cells}:layout={layout}[stacked]' + + f';[stacked]scale={target_w}:{target_h}:force_original_aspect_ratio=decrease,' + + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[out]' + ) + + # Audio + if mix_all_audio and n > 1: + audio_labels = [] + for i, src in enumerate(sources): + alabel = f'a{i}' + if src.has_audio: + filter_graph += ( + f';[{i}:a]apad=whole_dur={duration},' + f'atrim=0:{duration}[{alabel}]' + ) + else: + filter_graph += ( + f';anullsrc=r=44100:cl=stereo[null{i}]' + f';[null{i}]atrim=0:{duration}[{alabel}]' + ) + audio_labels.append(f'[{alabel}]') + filter_graph += ( + ';' + ''.join(audio_labels) + + f'amix=inputs={n}:duration=longest:normalize=0[aout]' + ) + audio_map = ['-map', '[aout]'] + else: + audio_map = ['-map', '0:a?'] + + cmd = [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-t', str(duration), + '-filter_complex', filter_graph, + '-map', '[out]', *audio_map, + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + output_path, + ] + self.ffmpeg.run(cmd, description=f'grid {n} webcams, {duration:.0f}s') + return output_path + + +class PresenterLayout: + """Composites presenter layout: main full-screen, optional PIP top-right.""" + + def __init__(self, ffmpeg: FFmpegRunner): + self.ffmpeg = ffmpeg + + def composite(self, main: GridSource, + overlay: Optional[GridSource], + extra_audio: List[GridSource], + duration: float, output_path: str, + target_w: int, target_h: int) -> str: + """Composite presenter layout. Output is exactly target_w x target_h.""" + pip_w = _even(target_w // 4) + pip_h = _even(target_h // 4) + margin = 16 + + inputs = ['-ss', str(main.offset), '-i', main.path] + all_sources = [main] + + if overlay: + inputs.extend(['-ss', str(overlay.offset), '-i', overlay.path]) + all_sources.append(overlay) + + for src in extra_audio: + inputs.extend(['-ss', str(src.offset), '-i', src.path]) + all_sources.append(src) + + # Video filter + vfilter = ( + f'[0:v]scale={target_w}:{target_h}:' + f'force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[main]' + ) + + if overlay: + vfilter += ( + f';[1:v]scale={pip_w}:{pip_h}:' + f'force_original_aspect_ratio=decrease,' + f'pad={pip_w}:{pip_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1[pip]' + f';[main][pip]overlay=x={target_w - pip_w - margin}:y={margin}[out]' + ) + else: + vfilter += ';[main]copy[out]' + + # Audio + afilter, audio_map = _build_audio_filter(all_sources, duration) + filter_graph = vfilter + afilter + + cmd = [ + 'ffmpeg', '-y', '-v', 'error', + *inputs, + '-t', str(duration), + '-filter_complex', filter_graph, + '-map', '[out]', *audio_map, + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + output_path, + ] + desc = 'presenter' + if overlay: + desc += ' + PIP' + if extra_audio: + desc += f' + {len(extra_audio)} audio' + self.ffmpeg.run(cmd, description=f'{desc}, {duration:.0f}s') + return output_path + + +class GridCompositor: + """Facade that delegates to GridLayout or PresenterLayout.""" + + def __init__(self, ffmpeg: FFmpegRunner): + self.ffmpeg = ffmpeg + self.grid = GridLayout(ffmpeg) + self.presenter = PresenterLayout(ffmpeg) + + # Expose GridLayout helpers for backward compat + compute_layout = staticmethod(GridLayout.compute_layout) + even = staticmethod(_even) + + def composite(self, active_segments: List[Union[GridSource, tuple]], + duration: float, output_path: str, + target_w: int, target_h: int, + mix_all_audio: bool = False) -> str: + # Normalize to GridSource objects + sources = [] + for seg in active_segments: + if isinstance(seg, GridSource): + sources.append(seg) + elif len(seg) == 3: + sources.append(GridSource(path=seg[0], offset=seg[1], has_audio=seg[2])) + else: + sources.append(GridSource(path=seg[0], offset=seg[1])) + return self.grid.composite(sources, duration, output_path, + target_w, target_h, mix_all_audio) + + def presenter_composite(self, main: GridSource, + overlay: Optional[GridSource], + extra_audio: List[GridSource], + duration: float, output_path: str, + target_w: int, target_h: int, + mix_all_audio: bool = True) -> str: + return self.presenter.composite(main, overlay, extra_audio, + duration, output_path, + target_w, target_h) diff --git a/mtslinker/prober.py b/mtslinker/prober.py new file mode 100644 index 0000000..6c95ef2 --- /dev/null +++ b/mtslinker/prober.py @@ -0,0 +1,109 @@ +import json +import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import List, Tuple + + +class MediaProber: + """Probes media files for stream info, duration, resolution, audio levels.""" + + def probe_streams(self, file_path: str) -> dict: + result = subprocess.run( + ['ffprobe', '-v', 'quiet', '-print_format', 'json', + '-show_streams', '-show_format', file_path], + capture_output=True, text=True, + ) + if result.returncode != 0: + return {} + return json.loads(result.stdout) + + def has_video(self, file_path: str) -> bool: + info = self.probe_streams(file_path) + return any(s.get('codec_type') == 'video' for s in info.get('streams', [])) + + def has_audio(self, file_path: str) -> bool: + info = self.probe_streams(file_path) + return any(s.get('codec_type') == 'audio' for s in info.get('streams', [])) + + def get_duration(self, file_path: str) -> float: + info = self.probe_streams(file_path) + fmt = info.get('format', {}) + if 'duration' in fmt: + return float(fmt['duration']) + for stream in info.get('streams', []): + if 'duration' in stream: + return float(stream['duration']) + return 0.0 + + def get_video_params(self, file_path: str) -> Tuple[int, int, str]: + info = self.probe_streams(file_path) + for stream in info.get('streams', []): + if stream.get('codec_type') == 'video': + w = int(stream.get('width', 1920)) + h = int(stream.get('height', 1080)) + pix_fmt = stream.get('pix_fmt', 'yuv420p') + return w, h, pix_fmt + return 1920, 1080, 'yuv420p' + + def probe_file(self, file_path: str) -> dict: + info = self.probe_streams(file_path) + streams = info.get('streams', []) + fmt = info.get('format', {}) + + has_video = any(s.get('codec_type') == 'video' for s in streams) + has_audio = any(s.get('codec_type') == 'audio' for s in streams) + + width, height, pix_fmt = 0, 0, 'yuv420p' + for s in streams: + if s.get('codec_type') == 'video': + width = int(s.get('width', 0)) + height = int(s.get('height', 0)) + pix_fmt = s.get('pix_fmt', 'yuv420p') + break + + duration = 0.0 + if 'duration' in fmt: + duration = float(fmt['duration']) + else: + for s in streams: + if 'duration' in s: + duration = float(s['duration']) + break + + valid = bool(streams) and duration > 0 + return { + 'path': file_path, + 'valid': valid, + 'has_video': has_video, + 'has_audio': has_audio, + 'width': width, + 'height': height, + 'pix_fmt': pix_fmt, + 'duration': duration, + } + + def probe_all_files(self, downloaded_files: list) -> List[dict]: + results = [] + items = [ + (item[0], item[1], + item[2] if len(item) > 2 else None, + item[3] if len(item) > 3 else False, + item[4] if len(item) > 4 else 0) + for item in downloaded_files + ] + with ThreadPoolExecutor(max_workers=8) as pool: + futures = {} + for path, start_time, conf_id, is_admin, api_duration in items: + fut = pool.submit(self.probe_file, path) + futures[fut] = (start_time, conf_id, is_admin, api_duration) + for fut in as_completed(futures): + start_time, conf_id, is_admin, api_duration = futures[fut] + info = fut.result() + info['start_time'] = start_time + info['conf_id'] = conf_id + info['is_admin'] = is_admin + if api_duration > 0: + info['api_duration'] = api_duration + results.append(info) + results.sort(key=lambda x: x['start_time']) + return results diff --git a/mtslinker/processor.py b/mtslinker/processor.py index ebd05c6..fc6052d 100644 --- a/mtslinker/processor.py +++ b/mtslinker/processor.py @@ -1,115 +1,515 @@ +"""Video processor — two-phase pipeline: analyze then execute. + +Uses composition of specialized classes: + FFmpegRunner — ffmpeg execution, GPU detection + MediaProber — file probing, duration, streams + SegmentBuilder — normalize, gaps, dedup + GridCompositor — multi-webcam grid layout + SlideCompositor — presentation slide overlay + AudioMerger — batched audio mixing +""" +import json import logging import os -from typing import Dict, Tuple, List, Union +import shutil +import subprocess +from typing import List + +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.segments import SegmentBuilder +from mtslinker.grid import GridCompositor +from mtslinker.slides import SlideCompositor +from mtslinker.audio import AudioMerger +from mtslinker.timeline import StreamTimeline, GridSource, AudioTrack + + +class VideoProcessor: + """Orchestrates the two-phase video processing pipeline.""" + + def __init__(self): + self.ffmpeg = FFmpegRunner() + self.prober = MediaProber() + self.segments = SegmentBuilder(self.ffmpeg, self.prober) + self.grid = GridCompositor(self.ffmpeg) + self.slides = SlideCompositor(self.ffmpeg) + self.audio = AudioMerger(self.ffmpeg, self.prober) + + def analyze(self, total_duration, downloaded_files, directory, + output_path, max_duration=None, slide_events=None, + timeline=None): + """Phase 1: Probe all files, make all decisions, write manifest. + + Uses the StreamTimeline built from mediasession events as the source + of truth — matches what the JS player does. + """ + logging.info('Phase 1: Analyzing files...') + + all_files = self.prober.probe_all_files(downloaded_files) + + video_files = [] + audio_files = [] + skipped = 0 + errors = [] + + for f in all_files: + if not f['valid']: + skipped += 1 + errors.append(f'Corrupt/invalid: {f["path"]}') + continue + if f['has_video']: + video_files.append(f) + else: + audio_files.append(f) + + if skipped: + logging.warning(f'Skipped {skipped} corrupt/invalid files') + logging.info(f'Segments: {len(video_files)} video, {len(audio_files)} audio-only') + + if not video_files: + raise ValueError('No valid video segments found.') + + # Determine target resolution + target_w, target_h, target_pix_fmt = 0, 0, 'yuv420p' + for f in video_files: + if f['width'] * f['height'] > target_w * target_h: + target_w, target_h, target_pix_fmt = f['width'], f['height'], f['pix_fmt'] + if target_w * target_h < 640 * 360: + target_w, target_h = 640, 360 + MAX_H = 720 + if target_h > MAX_H: + target_w = int(target_w * MAX_H / target_h) + target_w -= target_w % 2 + target_h = MAX_H + logging.info(f'Target resolution: {target_w}x{target_h}, pix_fmt={target_pix_fmt}') + + # Build segment plan from timeline (mediasession events). + # Each stream gets an independent playback element, all active + # streams play simultaneously — no dedup/overlap guessing needed. + segments = [] + timeline.map_downloaded_files(downloaded_files) + tw_windows = timeline.get_windows_with_files() + path_lookup = {f['path']: f for f in video_files} + + for tw in tw_windows: + sources = [] + for stream in tw.streams: + if not stream.file_path or stream.file_path not in path_lookup: + continue + offset = max(0, tw.start_time - stream.start_time) + f = path_lookup[stream.file_path] + is_admin = stream.conf_id in timeline.admin_conf_ids + sources.append({ + 'path': stream.file_path, + 'offset': offset, + 'remaining': tw.duration, + 'has_audio': stream.has_audio and f['has_audio'], + 'is_admin': is_admin, + 'is_screenshare': stream.is_screenshare, + }) + if not sources: + segments.append({ + 'type': 'gap', + 'duration': tw.duration, + 'start_time': tw.start_time, + }) + elif len(sources) == 1: + s = sources[0] + f = path_lookup[s['path']] + segments.append({ + 'type': 'video', + 'source_path': s['path'], + 'start_time': tw.start_time, + 'source_offset': s['offset'], + 'source_duration': f['duration'], + 'planned_duration': tw.duration, + 'has_audio': s['has_audio'], + 'width': f['width'], + 'height': f['height'], + }) + else: + # Presenter layout: admin/screenshare as main, others as PIP + screenshares = [s for s in sources if s.get('is_screenshare')] + admins = [s for s in sources if s.get('is_admin') and not s.get('is_screenshare')] + others = [s for s in sources if not s.get('is_admin') and not s.get('is_screenshare')] + + main_source = None + overlay_source = None + if screenshares: + main_source = screenshares[0] + overlay_source = admins[0] if admins else (others[0] if others else None) + elif admins: + main_source = admins[0] + overlay_source = others[0] if others else None + + if main_source: + used = {id(main_source)} + if overlay_source: + used.add(id(overlay_source)) + extra_audio = [s for s in sources if id(s) not in used and s.get('has_audio')] + segments.append({ + 'type': 'presenter', + 'start_time': tw.start_time, + 'planned_duration': tw.duration, + 'main_source': main_source, + 'overlay_source': overlay_source, + 'extra_audio_sources': extra_audio, + 'mix_all_audio': True, + }) + else: + # No admin, no screenshare — fall back to grid + segments.append({ + 'type': 'grid', + 'start_time': tw.start_time, + 'planned_duration': tw.duration, + 'sources': sources, + 'mix_all_audio': True, + }) + + # Fill leading/trailing gaps + filled = [] + current_time = 0.0 + for s in segments: + start = s.get('start_time', 0) + if start - current_time > 0.1: + filled.append({ + 'type': 'gap', + 'duration': start - current_time, + 'start_time': current_time, + }) + filled.append(s) + current_time = start + s.get('planned_duration', s.get('duration', 0)) + if current_time < total_duration - 0.1: + filled.append({ + 'type': 'gap', + 'duration': total_duration - current_time, + 'start_time': current_time, + }) + segments = filled + + logging.info(f'Timeline: {len(segments)} segments from mediasession events') + + # Audio tracks — only audio-only files (webcam audio is mixed + # inline by GridCompositor, no separate extraction needed) + all_audio = [{'path': f['path'], 'start_time': f['start_time'], + 'origin': 'original'} for f in audio_files] + + # Slide compositing plan + slide_plan = None + if slide_events: + CHUNK_SECS = 1800 + n_chunks = max(1, int(total_duration + CHUNK_SECS - 1) // CHUNK_SECS) + slide_plan = { + 'events': slide_events, + 'chunk_seconds': CHUNK_SECS, + 'num_chunks': n_chunks, + } + + # GPU detection + self.ffmpeg.detect_gpu() + gpu = { + 'nvenc': bool(self.ffmpeg.nvenc_available), + 'cuda_overlay': bool(self.ffmpeg.cuda_overlay_available), + } + + # Validation warnings + warnings = [] + for seg in segments: + if seg['type'] != 'video': + continue + src_dur = seg.get('source_duration', 0) + planned = seg.get('planned_duration', 0) + if src_dur and planned and src_dur < planned - 1.0: + warnings.append( + f'Segment at {seg["start_time"]:.0f}s: source={src_dur:.0f}s ' + f'but planned={planned:.0f}s (will pad {planned-src_dur:.0f}s black)' + ) + + manifest = { + 'version': 1, + 'total_duration': total_duration, + 'directory': directory, + 'output_path': output_path, + 'max_duration': max_duration, + 'target': {'width': target_w, 'height': target_h, 'pix_fmt': target_pix_fmt}, + 'overlap_strategy': 'timeline', + 'segments': segments, + 'audio_tracks': all_audio, + 'slide_compositing': slide_plan, + 'gpu': gpu, + 'errors': errors, + 'warnings': warnings, + 'stats': { + 'total_files': len(all_files), + 'video_files': len(video_files), + 'audio_files': len(audio_files), + 'kept_video': 0, + 'extra_audio': 0, + 'skipped': skipped, + 'segments': len(segments), + 'gaps': sum(1 for s in segments if s['type'] == 'gap'), + }, + } + + manifest_path = os.path.join(directory, 'manifest.json') + with open(manifest_path, 'w') as f: + json.dump(manifest, f, indent=2, ensure_ascii=False) + logging.info(f'Manifest written to {manifest_path}') + logging.info(f'Plan: {manifest["stats"]["segments"]} segments ' + f'({manifest["stats"]["gaps"]} gaps), ' + f'{len(all_audio)} audio tracks, ' + f'strategy=timeline') -import numpy as np -from moviepy.audio.AudioClip import AudioArrayClip, CompositeAudioClip -from moviepy.audio.io.AudioFileClip import AudioFileClip -from moviepy.video.VideoClip import ColorClip -from moviepy import VideoFileClip, concatenate_videoclips + if errors: + for e in errors: + logging.warning(f'Analyze: {e}') + if warnings: + for w in warnings: + logging.warning(f'Analyze: {w}') -import warnings -warnings.simplefilter("ignore") + return manifest + def execute(self, manifest): + """Phase 2: Execute the plan from the manifest.""" + logging.info('Phase 2: Executing plan...') -from mtslinker.downloader import download_video_chunk + directory = manifest['directory'] + output_path = manifest['output_path'] + total_duration = manifest['total_duration'] + target = manifest['target'] + target_w, target_h = target['width'], target['height'] + target_pix_fmt = target['pix_fmt'] + tmp_dir = os.path.join(directory, '_tmp_ffmpeg') + os.makedirs(tmp_dir, exist_ok=True) -def process_video_clips(directory: str, json_data: Dict) -> Tuple[float, List[VideoFileClip], List[AudioFileClip]]: + # Collect audio-only tracks (webcam audio is mixed inline by grid) + audio_files = [ + AudioTrack(path=t['path'], start_time=t['start_time'], origin=t['origin']) + for t in manifest['audio_tracks'] + ] + + # Process segments + segments = manifest['segments'] + grid_dir = os.path.join(tmp_dir, 'grid_segments') + concat_segments = [] + + for i, seg in enumerate(segments): + if seg['type'] == 'gap': + gap_path = os.path.join(tmp_dir, f'gap_{i}.mp4') + self.segments.generate_black(gap_path, seg['duration'], + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + logging.info(f'Generated {seg["duration"]:.1f}s black gap') + + elif seg['type'] == 'grid': + os.makedirs(grid_dir, exist_ok=True) + seg_path = os.path.join(grid_dir, f'grid_{i}.mp4') + duration = seg['planned_duration'] + sources = sorted(seg['sources'], + key=lambda s: not s.get('has_audio', False)) + mix_audio = seg.get('mix_all_audio', False) + active = [ + GridSource(path=s['path'], offset=s['offset'], + has_audio=s.get('has_audio', True)) + for s in sources + ] + try: + if len(active) == 1: + src = active[0] + self.ffmpeg.run( + ['ffmpeg', '-y', '-v', 'error', + '-ss', str(src.offset), '-i', src.path, + '-t', str(duration), + '-vf', f'scale={target_w}:{target_h}:' + f'force_original_aspect_ratio=decrease,' + f'pad={target_w}:{target_h}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', seg_path], + description=f'grid {i} (1 webcam, {duration:.0f}s)', + ) + else: + self.grid.composite(active, duration, seg_path, + target_w, target_h, + mix_all_audio=mix_audio) + with_audio = os.path.join(tmp_dir, f'grida_{i}.mp4') + final_seg = self.segments.ensure_audio(seg_path, with_audio) + concat_segments.append(final_seg) + except subprocess.CalledProcessError: + logging.warning(f'Grid segment {i} failed, using black gap') + gap_path = os.path.join(tmp_dir, f'gridfail_{i}.mp4') + self.segments.generate_black(gap_path, duration, + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + + elif seg['type'] == 'presenter': + os.makedirs(grid_dir, exist_ok=True) + seg_path = os.path.join(grid_dir, f'presenter_{i}.mp4') + duration = seg['planned_duration'] + ms = seg['main_source'] + main = GridSource(path=ms['path'], offset=ms['offset'], + has_audio=ms.get('has_audio', True), + is_admin=ms.get('is_admin', False)) + overlay = None + if seg.get('overlay_source'): + os_ = seg['overlay_source'] + overlay = GridSource(path=os_['path'], offset=os_['offset'], + has_audio=os_.get('has_audio', True), + is_admin=os_.get('is_admin', False)) + extra = [ + GridSource(path=s['path'], offset=s['offset'], + has_audio=s.get('has_audio', True)) + for s in seg.get('extra_audio_sources', []) + ] + try: + self.grid.presenter_composite( + main, overlay, extra, duration, seg_path, + target_w, target_h, + mix_all_audio=seg.get('mix_all_audio', True), + ) + with_audio = os.path.join(tmp_dir, f'presenta_{i}.mp4') + final_seg = self.segments.ensure_audio(seg_path, with_audio) + concat_segments.append(final_seg) + except subprocess.CalledProcessError: + logging.warning(f'Presenter segment {i} failed, using black gap') + gap_path = os.path.join(tmp_dir, f'presfail_{i}.mp4') + self.segments.generate_black(gap_path, duration, + target_w, target_h, target_pix_fmt) + concat_segments.append(gap_path) + + elif seg['type'] == 'video': + norm_path = os.path.join(tmp_dir, f'norm_{i}.mp4') + source_offset = seg.get('source_offset', 0) + planned_dur = seg.get('planned_duration', 0) + try: + self.segments.normalize( + seg['source_path'], norm_path, + target_w, target_h, target_pix_fmt, + max_duration=planned_dur, + seek=source_offset, + ) + with_audio_path = os.path.join(tmp_dir, f'norma_{i}.mp4') + final_seg = self.segments.ensure_audio( + norm_path, with_audio_path) + except subprocess.CalledProcessError: + logging.warning( + f'Video segment {i} failed, using black gap') + gap_dur = planned_dur or seg.get('source_duration', 0) + gap_path = os.path.join(tmp_dir, f'vidfail_{i}.mp4') + self.segments.generate_black(gap_path, gap_dur, + target_w, target_h, + target_pix_fmt) + concat_segments.append(gap_path) + continue + + actual_dur = self.prober.get_duration(final_seg) + if planned_dur > 0 and actual_dur < planned_dur - 1.0: + shortfall = planned_dur - actual_dur + logging.warning( + f'Segment {i} is {actual_dur:.1f}s but planned ' + f'{planned_dur:.1f}s — padding {shortfall:.1f}s' + ) + pad_path = os.path.join(tmp_dir, f'pad_{i}.mp4') + self.segments.generate_black(pad_path, shortfall, + target_w, target_h, target_pix_fmt) + concat_segments.append(final_seg) + concat_segments.append(pad_path) + else: + concat_segments.append(final_seg) + + # Concatenate + concat_list_path = os.path.join(tmp_dir, 'concat.txt') + with open(concat_list_path, 'w') as f: + for seg in concat_segments: + f.write(f"file '{os.path.abspath(seg)}'\n") + + video_only_path = os.path.join(tmp_dir, 'video_concat.mp4') + logging.info(f'Concatenating {len(concat_segments)} segments...') + + cap = manifest.get('max_duration') or total_duration + concat_cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-f', 'concat', '-safe', '0', + '-i', concat_list_path, '-c', 'copy', + ] + if cap: + concat_cmd.extend(['-t', str(cap)]) + concat_cmd.append(video_only_path) + self.ffmpeg.run(concat_cmd, description=f'concat {len(concat_segments)} segments') + + # Composite slides + slide_plan = manifest.get('slide_compositing') + if slide_plan and slide_plan.get('events'): + composited_path = os.path.join(tmp_dir, 'video_composited.mp4') + self.slides.composite( + video_only_path, slide_plan['events'], composited_path, + tmp_dir, total_duration, + ) + video_only_path = composited_path + + # Merge audio + if audio_files: + logging.info(f'Merging {len(audio_files)} audio tracks...') + result_path = self.audio.merge( + video_only_path, audio_files, tmp_dir, output_path, + total_duration, + ) + else: + shutil.move(video_only_path, output_path) + result_path = output_path + + shutil.rmtree(tmp_dir, ignore_errors=True) + logging.info(f'Final video saved to {result_path}') + return result_path + + def process(self, total_duration, downloaded_files, directory, + output_path, max_duration=None, slide_events=None, + timeline=None): + """Full pipeline: analyze then execute.""" + manifest = self.analyze( + total_duration, downloaded_files, directory, output_path, + max_duration, slide_events, timeline=timeline, + ) + return self.execute(manifest) + + +# --- Backward-compatible module-level functions --- + +def process_and_download_clips(directory, json_data): + """Parse API JSON for chunks, slides, admin IDs. + + Returns: + (total_duration, tagged_chunks, deduped_slides, timeline) + where timeline is a StreamTimeline instance built from mediasession events. + tagged_chunks are DownloadChunk-compatible tuples for backward compatibility + with the download pipeline. + """ total_duration = float(json_data.get('duration', 0)) if not total_duration: raise ValueError('Duration not found in JSON data.') - video_clips = [] - audio_clips = [] + timeline = StreamTimeline() + timeline.build(json_data) - for event in json_data.get('eventLogs', []): - if isinstance(event, dict): - data = event.get('data', {}) - if isinstance(data, dict) and 'url' in data: - url = data['url'] - start_time = event.get('relativeTime', 0) + # Convert to tuples for backward compatibility with download_chunks_parallel + tagged_chunks = [ + (c.url, c.start_time, c.conf_id, c.is_admin, c.api_duration) + for c in timeline.get_download_chunks() + ] + + deduped_slides = timeline.get_slide_events_as_dicts() + + return total_duration, tagged_chunks, deduped_slides, timeline + + +def compile_final_video(total_duration, downloaded_files, directory, + output_path, max_duration, slide_events=None, + timeline=None): + """Backward-compatible entry point.""" + processor = VideoProcessor() + processor.process(total_duration, downloaded_files, directory, + output_path, max_duration, slide_events, + timeline=timeline) - downloaded_file_path = download_video_chunk(url, directory) - try: - video_clip = VideoFileClip(downloaded_file_path, fps_source='fps').with_start(start_time) - video_clips.append(video_clip) - except (KeyError, OSError): - audio_clip = AudioFileClip(downloaded_file_path).with_start(start_time) - audio_clips.append(audio_clip) - logging.info(f'Total duration of clips: {total_duration}') - - return total_duration, video_clips, audio_clips - - -def create_video_with_gaps(total_duration: float, video_clips: List[VideoFileClip]) -> VideoFileClip: - clips = [] - current_time = 0.0 - - for video in video_clips: - if video.start > current_time: - gap_duration = video.start - current_time - if gap_duration > 0: - empty_clip = ColorClip(size=(1920, 1080), color=(0, 0, 0), duration=gap_duration).with_start( - current_time) - clips.append(empty_clip) - - clips.append(video) - current_time = video.end - - if current_time < total_duration: - remaining_duration = total_duration - current_time - empty_clip = ColorClip(size=(1920, 1080), color=(0, 0, 0), duration=remaining_duration).with_start(current_time) - clips.append(empty_clip) - - final_video = concatenate_videoclips(clips, method='compose') - logging.info(f'Final video duration: {final_video.duration}') - return final_video - - -def create_audio_with_gaps(total_duration: float, audio_clips: List[AudioFileClip]) -> CompositeAudioClip: - audio_segments = [] - current_time = 0 - - for audio in audio_clips: - if audio.start > current_time: - gap_duration = audio.start - current_time - if gap_duration > 0: - silence_segment = AudioArrayClip(np.zeros((int(gap_duration * 8000), 2)), fps=8000).with_start( - current_time) - audio_segments.append(silence_segment) - - audio_segments.append(audio) - current_time = audio.end - - if current_time < total_duration: - remaining_duration = total_duration - current_time - silence_segment = AudioArrayClip(np.zeros((int(remaining_duration * 8000), 2)), fps=8000).with_start( - current_time) - audio_segments.append(silence_segment) - - final_audio = CompositeAudioClip(audio_segments) - logging.info(f'Total audio duration: {final_audio.duration}') - return final_audio - - -def compile_final_video(total_duration: float, video_clips: List[VideoFileClip], audio_clips: List[AudioFileClip], - output_path: str, max_duration: Union[int, None]): - video_result = create_video_with_gaps(total_duration, video_clips) - - if audio_clips: - combined_audio = create_audio_with_gaps(total_duration, audio_clips) - video_result = video_result.with_audio(combined_audio) - - if max_duration: - if video_result.duration > max_duration: - logging.info(f'Duration limit! Crop!') - video_result = video_result.subclip(0, max_duration) - - video_result.write_videofile( - output_path, - codec='libx264', - audio_codec='aac', - preset='ultrafast', - threads=os.cpu_count() - ) diff --git a/mtslinker/segments.py b/mtslinker/segments.py new file mode 100644 index 0000000..95a4286 --- /dev/null +++ b/mtslinker/segments.py @@ -0,0 +1,105 @@ +import os +import subprocess + +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber + + +class SegmentBuilder: + """Builds, normalizes, and deduplicates video segments.""" + + def __init__(self, ffmpeg: FFmpegRunner, prober: MediaProber): + self.ffmpeg = ffmpeg + self.prober = prober + + def generate_black(self, output_path: str, duration: float, + width: int = 1920, height: int = 1080, + pix_fmt: str = 'yuv420p') -> str: + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', f'color=c=black:s={width}x{height}:d={duration}:r=25', + '-f', 'lavfi', '-i', f'anullsrc=r=44100:cl=stereo', + '-t', str(duration), + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', pix_fmt, + '-c:a', 'aac', '-b:a', '128k', + '-shortest', + output_path, + ], + description=f'generate black segment ({duration:.1f}s)', + ) + return output_path + + def ensure_audio(self, input_path: str, output_path: str) -> str: + info = self.prober.probe_streams(input_path) + has_audio = any( + s.get('codec_type') == 'audio' for s in info.get('streams', []) + ) + if has_audio: + return input_path + # anullsrc MUST be bounded by an input-side -t equal to the video + # length. With an unbounded anullsrc and -c:v copy, if the input has + # no decodable video the muxer waits forever for a video packet to + # interleave against, buffering silent audio without bound until + # "av_interleaved_write_frame: Cannot allocate memory" (observed in + # production on a normalize() output that seeked past end-of-source). + # A zero/unknown duration means the input is empty or corrupt, so + # there is no safe silent track to synthesize: fail loudly (the + # callers fall back to a black gap) instead of running unbounded. + duration = self.prober.get_duration(input_path) + if duration <= 0: + raise subprocess.CalledProcessError( + 1, 'ensure_audio', output=None, + stderr=(f'refusing silent-audio mux: {input_path} has no ' + f'positive duration (empty or corrupt input)'), + ) + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-i', input_path, + '-f', 'lavfi', '-t', str(duration), + '-i', 'anullsrc=r=44100:cl=stereo', + '-c:v', 'copy', '-c:a', 'aac', '-b:a', '128k', + '-shortest', '-max_interleave_delta', '0', + output_path, + ], + description='add silent audio stream', + ) + return output_path + + def normalize(self, input_path: str, output_path: str, + width: int, height: int, pix_fmt: str, + max_duration: float = 0, seek: float = 0) -> str: + seek_args = ['-ss', str(seek)] if seek > 0 else [] + duration_args = ['-t', str(max_duration)] if max_duration > 0 else [] + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + *seek_args, + '-i', input_path, + '-vf', f'scale={width}:{height}:force_original_aspect_ratio=decrease,' + f'pad={width}:{height}:(ow-iw)/2:(oh-ih)/2:black,setsar=1', + '-pix_fmt', pix_fmt, + *self.ffmpeg.get_video_encoder_fast(), + '-c:a', 'aac', '-b:a', '128k', '-ar', '44100', '-ac', '2', + '-r', '25', + *duration_args, + output_path, + ], + description=f'normalize segment {os.path.basename(input_path)}', + ) + # ffmpeg exits 0 even when -ss seeks past end-of-source and zero + # frames are decoded, leaving a tiny but valid empty container. + # Treat a zero-duration normalize as a failure so the caller falls + # back to a black gap instead of feeding an empty file into + # ensure_audio (whose muxer would then OOM). + if self.prober.get_duration(output_path) <= 0: + raise subprocess.CalledProcessError( + 1, 'normalize', output=None, + stderr=(f'normalize produced empty output for {input_path} ' + f'(seek {seek}s past end-of-source?)'), + ) + return output_path + + diff --git a/mtslinker/slides.py b/mtslinker/slides.py new file mode 100644 index 0000000..04b51bb --- /dev/null +++ b/mtslinker/slides.py @@ -0,0 +1,194 @@ +import logging +import os +import shutil +import subprocess +from typing import Dict, List + +from mtslinker.ffmpeg import FFmpegRunner + + +class SlideCompositor: + """Composites presentation slides alongside webcam video.""" + + CANVAS_W = 1280 + CANVAS_H = 720 + SLIDE_W = 960 + CAM_W = 320 + CHUNK_SECS = 1800 + # Exit codes that mean the kernel killed ffmpeg for memory. Python's + # subprocess reports SIGKILL as -9; some shells/wrappers surface it as + # 128+9 = 137. Both indicate the same OOM-kill scenario. + _OOM_KILL_CODES = (-9, 137) + + def __init__(self, ffmpeg: FFmpegRunner): + self.ffmpeg = ffmpeg + + def composite(self, video_path: str, slide_events: List[Dict], + output_path: str, tmp_dir: str, total_duration: float) -> str: + logging.info(f'Compositing {len(slide_events)} slide changes onto video...') + + slide_track_path = self._build_slide_track(slide_events, tmp_dir, total_duration) + + filter_graph = ( + f'[0:v]fps=25,scale={self.CAM_W}:-2,setsar=1[webcam];' + f'[1:v]fps=25,scale={self.SLIDE_W}:{self.CANVAS_H}:' + f'force_original_aspect_ratio=decrease,' + f'pad={self.SLIDE_W}:{self.CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white[slide];' + f'color=c=black:s={self.CANVAS_W}x{self.CANVAS_H}:r=25[bg];' + f'[bg][slide]overlay=0:0[tmp];' + f'[tmp][webcam]overlay={self.SLIDE_W}:0[out]' + ) + + n_chunks = max(1, int(total_duration + self.CHUNK_SECS - 1) // self.CHUNK_SECS) + + if n_chunks == 1: + cmd = [ + 'ffmpeg', '-y', '-v', 'warning', + '-i', video_path, '-i', slide_track_path, + '-filter_complex', filter_graph, + '-map', '[out]', '-map', '0:a?', + *self.ffmpeg.get_video_encoder(), + '-c:a', 'copy', '-r', '25', '-shortest', + output_path, + ] + self.ffmpeg.run(cmd, description='composite slides + webcam') + else: + logging.info(f'Compositing in {n_chunks} chunks of {self.CHUNK_SECS}s to avoid OOM') + chunks_dir = os.path.join(tmp_dir, 'composite_chunks') + os.makedirs(chunks_dir, exist_ok=True) + chunk_paths = [] + # NVENC keeps pinned host buffers per stream; on a long, heavy + # filter_complex graph the OOM-killer can SIGKILL ffmpeg mid + # chunk. Once that has happened, the NVENC budget for *this* + # video is over budget — stay on CPU for the rest of the chunks + # instead of paying the kill cost again on each one. + use_cpu_encoder = False + + def build_chunk_cmd(ss_arg, chunk_path_arg, enc_args): + return [ + 'ffmpeg', '-y', '-v', 'warning', + '-ss', str(ss_arg), '-t', str(self.CHUNK_SECS), + '-i', video_path, + '-ss', str(ss_arg), '-t', str(self.CHUNK_SECS), + '-i', slide_track_path, + '-filter_complex', filter_graph, + '-map', '[out]', '-map', '0:a?', + *enc_args, '-c:a', 'aac', '-b:a', '192k', + '-r', '25', '-shortest', + chunk_path_arg, + ] + + for ci in range(n_chunks): + ss = ci * self.CHUNK_SECS + chunk_path = os.path.join(chunks_dir, f'chunk_{ci:03d}.mp4') + label = f'composite chunk {ci+1}/{n_chunks}' + + if use_cpu_encoder: + self.ffmpeg.run( + build_chunk_cmd(ss, chunk_path, + self.ffmpeg.get_video_encoder_cpu()), + description=f'{label} (cpu)', + ) + else: + try: + self.ffmpeg.run( + build_chunk_cmd(ss, chunk_path, + self.ffmpeg.get_video_encoder()), + description=label, + ) + except subprocess.CalledProcessError as e: + if e.returncode not in self._OOM_KILL_CODES: + raise + logging.warning( + f'{label} OOM-killed (exit {e.returncode}); ' + f'retrying this chunk with CPU encoder and ' + f'falling back to CPU for any remaining chunks' + ) + use_cpu_encoder = True + self.ffmpeg.run( + build_chunk_cmd(ss, chunk_path, + self.ffmpeg.get_video_encoder_cpu()), + description=f'{label} (cpu retry)', + ) + chunk_paths.append(chunk_path) + logging.info(f'Composite chunk {ci+1}/{n_chunks} done') + + chunk_list = os.path.join(chunks_dir, 'concat.txt') + with open(chunk_list, 'w') as f: + for cp in chunk_paths: + f.write(f"file '{os.path.abspath(cp)}'\n") + self.ffmpeg.run( + ['ffmpeg', '-y', '-v', 'error', + '-f', 'concat', '-safe', '0', '-i', chunk_list, + '-c', 'copy', output_path], + description='concat composite chunks', + ) + shutil.rmtree(chunks_dir, ignore_errors=True) + + logging.info('Slide compositing complete') + slides_dir = os.path.join(tmp_dir, 'slide_segments') + shutil.rmtree(slides_dir, ignore_errors=True) + return output_path + + def _build_slide_track(self, slide_events, tmp_dir, total_duration): + slides_dir = os.path.join(tmp_dir, 'slide_segments') + os.makedirs(slides_dir, exist_ok=True) + + slide_segments = [] + for i, se in enumerate(slide_events): + t_start = se['time'] + t_end = slide_events[i + 1]['time'] if i + 1 < len(slide_events) else total_duration + duration = t_end - t_start + if duration <= 0: + continue + + seg_path = os.path.join(slides_dir, f'seg_{i}.mp4') + n_frames = max(1, int(duration)) + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-loop', '1', '-framerate', '1', + '-i', se['local_path'], + '-vf', f'scale={self.SLIDE_W}:{self.CANVAS_H}:' + f'force_original_aspect_ratio=decrease,' + f'pad={self.SLIDE_W}:{self.CANVAS_H}:(ow-iw)/2:(oh-ih)/2:white', + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + '-r', '1', '-frames:v', str(n_frames), + seg_path, + ], + description=f'slide segment {i+1}/{len(slide_events)} ' + f'({n_frames} frames, {duration:.0f}s)', + ) + slide_segments.append(seg_path) + + first_time = slide_events[0]['time'] if slide_events else 0 + if first_time > 0.5: + leader_path = os.path.join(slides_dir, 'leader.mp4') + self.ffmpeg.run( + [ + 'ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', + f'color=c=black:s={self.SLIDE_W}x{self.CANVAS_H}:d={first_time}:r=1', + *self.ffmpeg.get_video_encoder_fast(), + '-pix_fmt', 'yuv420p', + leader_path, + ], + description='slide leader (black)', + ) + slide_segments.insert(0, leader_path) + + slide_track_path = os.path.join(tmp_dir, 'slide_track.mp4') + concat_list = os.path.join(slides_dir, 'concat.txt') + with open(concat_list, 'w') as f: + for seg in slide_segments: + f.write(f"file '{os.path.abspath(seg)}'\n") + + self.ffmpeg.run( + ['ffmpeg', '-y', '-v', 'error', + '-f', 'concat', '-safe', '0', '-i', concat_list, + '-c', 'copy', slide_track_path], + description='concat slide track', + ) + logging.info('Slide track created') + return slide_track_path diff --git a/mtslinker/timeline.py b/mtslinker/timeline.py new file mode 100644 index 0000000..2b78f7f --- /dev/null +++ b/mtslinker/timeline.py @@ -0,0 +1,413 @@ +"""StreamTimeline — builds a playback timeline from API mediasession events. + +Replicates what the MTS-Link web player does: uses mediasession.add/update +events to know exactly which files to play and when, and conference events +to know which streams have audio/video. +""" +import logging +from dataclasses import dataclass, field +from typing import Dict, List, Optional + + +@dataclass +class MediaSession: + """A single media session (one recorded file).""" + id: str + url: str + start_time: float + duration: float = 0.0 + conf_id: Optional[str] = None + has_audio: bool = True + has_video: bool = True + is_screenshare: bool = False + user_id: Optional[str] = None + file_path: Optional[str] = None # set after download + + +@dataclass +class SlideEvent: + """A presentation slide change event.""" + time: float + slide_number: int + slide_url: str + local_path: Optional[str] = None # set after download + + +@dataclass +class DownloadChunk: + """A media chunk to download.""" + url: str + start_time: float + conf_id: Optional[str] = None + is_admin: bool = False + api_duration: float = 0.0 + session_id: Optional[str] = None + + +@dataclass +class GridSource: + """A source stream for grid compositing.""" + path: str + offset: float + has_audio: bool = True + is_admin: bool = False + + +@dataclass +class AudioTrack: + """An audio track to merge into the final video.""" + path: str + start_time: float + origin: str = 'original' # 'original' or 'webcam_extract' + + +@dataclass +class TimeWindow: + """A time window where the set of active streams is constant.""" + start_time: float + duration: float + streams: List[MediaSession] = field(default_factory=list) + + @property + def end_time(self) -> float: + return self.start_time + self.duration + + @property + def has_video(self) -> bool: + return any(s.has_video for s in self.streams) + + @property + def has_audio(self) -> bool: + return any(s.has_audio for s in self.streams) + + @property + def stream_count(self) -> int: + return len(self.streams) + + +class StreamTimeline: + """Builds a timeline of active streams from API event logs. + + Mirrors the web player's approach: each mediasession gets an independent + playback element, and all active streams play simultaneously. + """ + + def __init__(self): + self.sessions: Dict[str, MediaSession] = {} + self.admin_conf_ids: set = set() + self.total_duration: float = 0.0 + self.slide_events: List[SlideEvent] = [] + + def build(self, json_data: dict) -> List[TimeWindow]: + """Parse API JSON and build timeline windows. + + Args: + json_data: Full API response from /api/eventsessions/{sid}/record + + Returns: + List of TimeWindow objects in chronological order. + """ + self.total_duration = float(json_data.get('duration', 0)) + self._extract_admin_info(json_data) + self._extract_sessions(json_data) + self._extract_slides(json_data) + windows = self._compute_windows() + return windows + + def _extract_admin_info(self, json_data: dict): + """Extract admin user IDs and their conference IDs.""" + admin_user_ids = set() + user_to_conf = {} + + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data_list = event.get('data', []) + if isinstance(data_list, dict): + data_list = [data_list] + if not isinstance(data_list, list): + continue + + for d in data_list: + if not isinstance(d, dict): + continue + if 'userlist' in module: + role = d.get('role', '') + user = d.get('user', {}) + if isinstance(user, dict) and role == 'ADMIN': + uid = user.get('id') + if uid: + admin_user_ids.add(uid) + if module == 'conference.add': + user = d.get('user', {}) + if isinstance(user, dict): + uid = user.get('id') + cid = d.get('id') + if uid and cid: + user_to_conf.setdefault(uid, set()).add(cid) + + for uid in admin_user_ids: + self.admin_conf_ids.update(user_to_conf.get(uid, set())) + + def _extract_sessions(self, json_data: dict): + """Extract mediasession.add/update events into MediaSession objects.""" + conf_users = {} + conf_props = {} + + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data = event.get('data', {}) + if not isinstance(data, dict): + continue + + if module == 'conference.add': + cid = data.get('id') + user = data.get('user', {}) + if isinstance(user, dict) and cid: + uid = user.get('id') + if uid: + conf_users[cid] = uid + conf_props[cid] = { + 'has_audio': data.get('hasAudio', True), + 'has_video': data.get('hasVideo', True), + } + + elif module == 'conference.update': + cid = data.get('id') + if cid and cid in conf_props: + if 'hasAudio' in data: + conf_props[cid]['has_audio'] = data['hasAudio'] + if 'hasVideo' in data: + conf_props[cid]['has_video'] = data['hasVideo'] + + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data = event.get('data', {}) + if not isinstance(data, dict): + continue + + if module == 'mediasession.add' and 'url' in data: + ms_id = data.get('id') + if not ms_id: + continue + conf_id = None + is_screenshare = False + stream = data.get('stream', {}) + if isinstance(stream, dict): + conf = stream.get('conference', {}) + if isinstance(conf, dict): + conf_id = conf.get('id') + ss = stream.get('screensharing', {}) + if isinstance(ss, dict) and ss.get('id'): + is_screenshare = True + + props = conf_props.get(conf_id, {}) + self.sessions[ms_id] = MediaSession( + id=ms_id, + url=data['url'], + start_time=event.get('relativeTime', 0), + conf_id=conf_id, + has_audio=props.get('has_audio', True), + has_video=props.get('has_video', True), + is_screenshare=is_screenshare, + user_id=conf_users.get(conf_id), + ) + + elif module == 'mediasession.update': + ms_id = data.get('id') + if ms_id and ms_id in self.sessions: + if 'duration' in data: + self.sessions[ms_id].duration = float(data['duration']) + + # Fallback: pick up URLs not covered by mediasession events + known_urls = {s.url for s in self.sessions.values()} + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + module = event.get('module', '') + data = event.get('data', {}) + if not isinstance(data, dict): + continue + if module == 'mediasession.add': + continue + url = data.get('url') + if url and url not in known_urls: + fallback_id = f'fallback_{len(self.sessions)}' + conf_id = None + stream = data.get('stream', {}) + if isinstance(stream, dict): + conf = stream.get('conference', {}) + if isinstance(conf, dict): + conf_id = conf.get('id') + self.sessions[fallback_id] = MediaSession( + id=fallback_id, + url=url, + start_time=event.get('relativeTime', 0), + conf_id=conf_id, + ) + known_urls.add(url) + + logging.info(f'Timeline: {len(self.sessions)} media sessions extracted') + + def _extract_slides(self, json_data: dict): + """Extract presentation.update events into SlideEvent objects.""" + raw_slides = [] + for event in json_data.get('eventLogs', []): + if not isinstance(event, dict): + continue + if event.get('module') != 'presentation.update': + continue + data = event.get('data', {}) + if not isinstance(data, dict): + continue + fr = data.get('fileReference', {}) + if not isinstance(fr, dict): + continue + slide = fr.get('slide', {}) + if not isinstance(slide, dict) or not slide.get('url'): + continue + raw_slides.append(SlideEvent( + time=event.get('relativeTime', 0), + slide_number=slide.get('number', 0), + slide_url=slide['url'], + )) + + # Deduplicate consecutive identical slides + self.slide_events = [] + for se in raw_slides: + if not self.slide_events or se.slide_url != self.slide_events[-1].slide_url: + self.slide_events.append(se) + + if self.slide_events: + logging.info(f'Timeline: {len(self.slide_events)} presentation slide changes') + + def _compute_windows(self) -> List[TimeWindow]: + """Compute time windows where the set of active streams is constant.""" + if not self.sessions: + return [] + + boundaries = set() + boundaries.add(0.0) + if self.total_duration > 0: + boundaries.add(self.total_duration) + + for s in self.sessions.values(): + boundaries.add(s.start_time) + end = s.start_time + s.duration if s.duration > 0 else self.total_duration + boundaries.add(end) + + sorted_times = sorted(boundaries) + + windows = [] + prev_streams_key = None + merge_start = None + merge_streams = None + + for i in range(len(sorted_times) - 1): + t_start = sorted_times[i] + t_end = sorted_times[i + 1] + if t_end - t_start < 0.1: + continue + + active = [] + for s in self.sessions.values(): + s_end = s.start_time + s.duration if s.duration > 0 else self.total_duration + if s.start_time < t_end and s_end > t_start: + active.append(s) + + streams_key = tuple(s.id for s in sorted(active, key=lambda x: x.id)) + + if streams_key == prev_streams_key and merge_start is not None: + continue + else: + if merge_start is not None: + windows.append(TimeWindow( + start_time=merge_start, + duration=t_start - merge_start, + streams=merge_streams, + )) + merge_start = t_start + merge_streams = active + prev_streams_key = streams_key + + if merge_start is not None and sorted_times: + windows.append(TimeWindow( + start_time=merge_start, + duration=sorted_times[-1] - merge_start, + streams=merge_streams, + )) + + windows = [w for w in windows if w.duration > 0.1] + + logging.info(f'Timeline: {len(windows)} time windows computed') + return windows + + def get_download_chunks(self) -> List[DownloadChunk]: + """Get list of chunks to download as DownloadChunk objects.""" + result = [] + for s in self.sessions.values(): + result.append(DownloadChunk( + url=s.url, + start_time=s.start_time, + conf_id=s.conf_id, + is_admin=s.conf_id in self.admin_conf_ids, + api_duration=s.duration, + session_id=s.id, + )) + return sorted(result, key=lambda x: x.start_time) + + def get_slide_events_as_dicts(self) -> List[dict]: + """Get slide events as dicts (compatible with existing download pipeline).""" + return [ + {'time': se.time, 'slide_number': se.slide_number, + 'slide_url': se.slide_url} + for se in self.slide_events + ] + + def map_downloaded_files(self, downloaded_files: list): + """Map downloaded files back to sessions. + + Args: + downloaded_files: List of DownloadedFile-like objects or + (path, start_time, conf_id, is_admin) tuples. + """ + time_conf_to_session = {} + for s in self.sessions.values(): + key = (round(s.start_time, 1), s.conf_id) + time_conf_to_session[key] = s + + for item in downloaded_files: + if hasattr(item, 'path'): + path, start_time, conf_id = item.path, item.start_time, item.conf_id + else: + path = item[0] + start_time = item[1] + conf_id = item[2] if len(item) > 2 else None + + key = (round(start_time, 1), conf_id) + session = time_conf_to_session.get(key) + if session: + session.file_path = path + else: + for s in self.sessions.values(): + if abs(s.start_time - start_time) < 0.5 and s.file_path is None: + s.file_path = path + break + + def get_windows_with_files(self) -> List[TimeWindow]: + """Get windows filtered to only include streams with downloaded files.""" + result = [] + for w in self._compute_windows(): + filed_streams = [s for s in w.streams if s.file_path] + if filed_streams: + result.append(TimeWindow( + start_time=w.start_time, + duration=w.duration, + streams=filed_streams, + )) + return result diff --git a/mtslinker/webinar.py b/mtslinker/webinar.py index 5bab43c..fffdc69 100644 --- a/mtslinker/webinar.py +++ b/mtslinker/webinar.py @@ -2,15 +2,20 @@ import os import re -from mtslinker.downloader import construct_json_data_url, fetch_json_data -from mtslinker.processor import compile_final_video, process_video_clips +from mtslinker.downloader import ( + construct_json_data_url, + fetch_json_data, + download_chunks_parallel, + download_slide_images, +) +from mtslinker.processor import compile_final_video, process_and_download_clips from mtslinker.utils import create_directory_if_not_exists def fetch_webinar_data(event_sessions: str, record_id: str, session_id=None, max_duration=None): json_data_url = construct_json_data_url(event_session_id=event_sessions, recording_id=record_id) json_data = fetch_json_data(url=json_data_url, session_id=session_id) - + if not json_data: logging.error('Failed to fetch webinar data. Check the session ID or URL.') return @@ -19,11 +24,22 @@ def fetch_webinar_data(event_sessions: str, record_id: str, session_id=None, max directory = create_directory_if_not_exists(sanitized_name) output_video_path = os.path.join(directory, f'{sanitized_name}.mp4') - total_duration, video_clips, audio_clips = process_video_clips(directory, json_data) - logging.info( - f'Downloaded and processed {len(video_clips) + len(audio_clips)} files ({total_duration} sec) for merging.') + total_duration, chunks, slide_events, timeline = process_and_download_clips(directory, json_data) + logging.info(f'Found {len(chunks)} chunks to download ({total_duration} sec total)') + + # Download all chunks in parallel + downloaded_files = download_chunks_parallel(chunks, directory) + logging.info(f'Downloaded {len(downloaded_files)} files, starting merge...') - compile_final_video(total_duration, video_clips, audio_clips, output_video_path, max_duration) + # Download presentation slides if any + downloaded_slides = [] + if slide_events: + downloaded_slides = download_slide_images(slide_events, directory) + + compile_final_video( + total_duration, downloaded_files, directory, output_video_path, + max_duration, slide_events=downloaded_slides, timeline=timeline, + ) logging.info(f'Final video saved to {output_video_path}') - + return 1 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..9573d48 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest>=7.0 diff --git a/requirements.txt b/requirements.txt index fd6c954..cd69fab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ tqdm==4.66.6 httpx==0.27.2 -moviepy==2.1.1 diff --git a/setup.py b/setup.py index dfb3283..d70641b 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,10 @@ setup( name='mtslinker', - version='1.0.0', + version='2.0.0', packages=find_packages(), install_requires=[ 'httpx>=0.27.2', - 'moviepy>=1.0.3', 'tqdm>=4.66.6', ], entry_points={ diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f0ef0c0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,54 @@ +import os +import shutil +import subprocess +import tempfile +import pytest + +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.segments import SegmentBuilder + + +@pytest.fixture +def ffmpeg(): + return FFmpegRunner() + + +@pytest.fixture +def prober(): + return MediaProber() + + +@pytest.fixture +def segments(ffmpeg, prober): + return SegmentBuilder(ffmpeg, prober) + + +@pytest.fixture +def tmp_dir(): + d = tempfile.mkdtemp() + yield d + shutil.rmtree(d, ignore_errors=True) + + +def make_test_video(path, duration=3, with_audio=True, width=320, height=240): + audio_args = [] + if with_audio: + audio_args = ['-f', 'lavfi', '-i', f'sine=frequency=440:duration={duration}', + '-c:a', 'aac', '-shortest'] + subprocess.run( + ['ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', f'color=red:s={width}x{height}:d={duration}:r=25', + *audio_args, + '-c:v', 'libx264', '-preset', 'ultrafast', path], + capture_output=True, check=True, + ) + + +def make_test_audio(path, duration=3, frequency=880): + subprocess.run( + ['ffmpeg', '-y', '-v', 'error', + '-f', 'lavfi', '-i', f'sine=frequency={frequency}:duration={duration}', + '-c:a', 'aac', path], + capture_output=True, check=True, + ) diff --git a/tests/test_audio_pipeline.py b/tests/test_audio_pipeline.py new file mode 100644 index 0000000..0884c9b --- /dev/null +++ b/tests/test_audio_pipeline.py @@ -0,0 +1,139 @@ +"""Integration tests for audio pipeline — catches echo, silence, timeline issues.""" +import os +import subprocess +from tests.conftest import make_test_video, make_test_audio +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.audio import AudioMerger +from mtslinker.grid import GridCompositor +from mtslinker.segments import SegmentBuilder + + +def _get_volume(path): + """Get mean volume in dB from a media file.""" + result = subprocess.run( + ['ffmpeg', '-i', path, '-af', 'volumedetect', '-f', 'null', '-'], + capture_output=True, text=True, + ) + for line in result.stderr.splitlines(): + if 'mean_volume' in line: + return float(line.split('mean_volume:')[1].strip().split()[0]) + return None + + +def test_grid_audio_no_echo(ffmpeg, prober, tmp_dir): + """Grid composite should not cause echo when audio is also merged.""" + # Create two webcams: v1 has audio (440Hz), v2 has no audio + v1 = os.path.join(tmp_dir, 'cam1.mp4') + v2 = os.path.join(tmp_dir, 'cam2.mp4') + make_test_video(v1, duration=3, with_audio=True) + make_test_video(v2, duration=3, with_audio=False) + + # Grid composite — should include v1's audio (input 0, sorted by has_audio) + grid = GridCompositor(ffmpeg) + grid_out = os.path.join(tmp_dir, 'grid.mp4') + grid.composite([(v1, 0), (v2, 0)], 3.0, grid_out, 640, 360) + + # Verify grid has audio + assert prober.has_audio(grid_out) + grid_vol = _get_volume(grid_out) + assert grid_vol is not None + assert grid_vol > -50 # should be audible + + # Now merge an audio-only track on top (different frequency) + audio_track = os.path.join(tmp_dir, 'extra.m4a') + make_test_audio(audio_track, duration=3, frequency=880) + + merger = AudioMerger(ffmpeg, prober) + final = os.path.join(tmp_dir, 'final.mp4') + merger.merge(grid_out, [(audio_track, 0.0)], tmp_dir, final, total_duration=3) + + final_vol = _get_volume(final) + assert final_vol is not None + # Final should be louder than grid alone (added extra audio) + # but NOT double the grid audio (no echo) + # If echo: grid_vol ~= -21, final_vol ~= -15 (6dB louder = doubled) + # Without echo: grid_vol ~= -21, final_vol ~= -18 (mixed with different freq) + assert final_vol > grid_vol - 1 # at least as loud + + +def test_grid_audio_from_first_input(ffmpeg, prober, tmp_dir): + """Grid should take audio from first input (sorted: audio-bearing first).""" + v_with = os.path.join(tmp_dir, 'with_audio.mp4') + v_without = os.path.join(tmp_dir, 'no_audio.mp4') + make_test_video(v_with, duration=2, with_audio=True) + make_test_video(v_without, duration=2, with_audio=False) + + grid = GridCompositor(ffmpeg) + out = os.path.join(tmp_dir, 'grid.mp4') + # Put audio-bearing first (as the processor does) + grid.composite([(v_with, 0), (v_without, 0)], 2.0, out, 640, 360) + + assert prober.has_audio(out) + vol = _get_volume(out) + assert vol is not None + assert vol > -50 + + +def test_audio_merge_preserves_timing(ffmpeg, prober, tmp_dir): + """Audio tracks placed with adelay should not shift.""" + # Create a 10s video + vid = os.path.join(tmp_dir, 'video.mp4') + make_test_video(vid, duration=10, with_audio=True) + + # Create audio that starts at 5s + audio = os.path.join(tmp_dir, 'delayed.m4a') + make_test_audio(audio, duration=3, frequency=880) + + merger = AudioMerger(ffmpeg, prober) + final = os.path.join(tmp_dir, 'final.mp4') + merger.merge(vid, [(audio, 5.0)], tmp_dir, final, total_duration=10) + + # Final should be 10s + dur = prober.get_duration(final) + assert 9.5 < dur < 10.5 + + # Audio at 0-4s should be original only + vol_start = _get_volume_at(final, 0, 3) + # Audio at 5-8s should be louder (original + delayed) + vol_mid = _get_volume_at(final, 5, 3) + # Both should have audio + assert vol_start is not None + assert vol_mid is not None + + +def _get_volume_at(path, start, duration): + """Get mean volume at a specific time range.""" + result = subprocess.run( + ['ffmpeg', '-ss', str(start), '-i', path, + '-t', str(duration), '-af', 'volumedetect', '-f', 'null', '-'], + capture_output=True, text=True, + ) + for line in result.stderr.splitlines(): + if 'mean_volume' in line: + return float(line.split('mean_volume:')[1].strip().split()[0]) + return None + + +def test_segment_duration_matches_plan(ffmpeg, prober, tmp_dir): + """Normalized segments should match planned duration within tolerance.""" + segments = SegmentBuilder(ffmpeg, prober) + vid = os.path.join(tmp_dir, 'input.mp4') + out = os.path.join(tmp_dir, 'norm.mp4') + make_test_video(vid, duration=10) + + # Normalize with max_duration=5 + segments.normalize(vid, out, 640, 360, 'yuv420p', max_duration=5) + dur = prober.get_duration(out) + assert 4.5 < dur < 5.5 + + +def test_black_segment_has_audio(ffmpeg, prober, tmp_dir): + """Black gap segments must have a silent audio stream for concat.""" + segments = SegmentBuilder(ffmpeg, prober) + path = os.path.join(tmp_dir, 'black.mp4') + segments.generate_black(path, 3.0, 640, 360) + assert prober.has_audio(path) + vol = _get_volume(path) + assert vol is not None + assert vol < -80 # should be silent diff --git a/tests/test_ffmpeg.py b/tests/test_ffmpeg.py new file mode 100644 index 0000000..da827e4 --- /dev/null +++ b/tests/test_ffmpeg.py @@ -0,0 +1,48 @@ +import subprocess +import pytest +from mtslinker.ffmpeg import FFmpegRunner + + +def test_detect_gpu_sets_flags(ffmpeg): + ffmpeg.detect_gpu() + assert ffmpeg.nvenc_available is not None + assert ffmpeg.cuda_overlay_available is not None + + +def test_detect_gpu_idempotent(ffmpeg): + ffmpeg.detect_gpu() + first = ffmpeg.nvenc_available + ffmpeg.detect_gpu() + assert ffmpeg.nvenc_available == first + + +def test_get_video_encoder_returns_codec_flag(ffmpeg): + enc = ffmpeg.get_video_encoder() + assert '-c:v' in enc + assert len(enc) >= 2 + + +def test_get_video_encoder_fast_returns_codec_flag(ffmpeg): + enc = ffmpeg.get_video_encoder_fast() + assert '-c:v' in enc + + +def test_get_video_encoder_cpu_always_returns_libx264(ffmpeg): + """The CPU fallback must never pick NVENC, even when NVENC is available. + + It exists specifically to bypass a system OOM-kill of NVENC, so it + cannot conditionally fall back to the very thing it is meant to + replace. + """ + enc = ffmpeg.get_video_encoder_cpu() + assert enc[enc.index('-c:v') + 1] == 'libx264' + + +def test_run_success(ffmpeg): + result = ffmpeg.run(['ffmpeg', '-version'], description='version') + assert result.returncode == 0 + + +def test_run_failure_raises(ffmpeg): + with pytest.raises(subprocess.CalledProcessError): + ffmpeg.run(['ffmpeg', '-i', '/nonexistent_file.mp4'], description='bad') diff --git a/tests/test_grid.py b/tests/test_grid.py new file mode 100644 index 0000000..1aa62ba --- /dev/null +++ b/tests/test_grid.py @@ -0,0 +1,192 @@ +import os +import subprocess +from mtslinker.grid import GridCompositor, GridLayout, PresenterLayout, _even, _build_audio_filter +from mtslinker.timeline import GridSource +from tests.conftest import make_test_video + + +# --- GridLayout unit tests --- + +def test_compute_layout_single(ffmpeg): + assert GridLayout.compute_layout(1) == (1, 1) + + +def test_compute_layout_two(ffmpeg): + assert GridLayout.compute_layout(2) == (2, 1) + + +def test_compute_layout_four(ffmpeg): + assert GridLayout.compute_layout(4) == (2, 2) + + +def test_compute_layout_five(ffmpeg): + assert GridLayout.compute_layout(5) == (3, 2) + + +def test_even(): + assert _even(640) == 640 + assert _even(641) == 640 + assert _even(1) == 0 + + +def test_grid_composite_two_webcams(ffmpeg, tmp_dir): + grid = GridLayout(ffmpeg) + v1 = os.path.join(tmp_dir, 'v1.mp4') + v2 = os.path.join(tmp_dir, 'v2.mp4') + out = os.path.join(tmp_dir, 'grid.mp4') + make_test_video(v1, duration=2) + make_test_video(v2, duration=2) + sources = [GridSource(v1, 0), GridSource(v2, 0)] + grid.composite(sources, 2.0, out, 640, 360) + assert os.path.exists(out) + assert os.path.getsize(out) > 0 + + +def test_grid_output_matches_target_resolution(ffmpeg, prober, tmp_dir): + """Grid output must be exactly target_w x target_h for concat safety.""" + grid = GridLayout(ffmpeg) + v1 = os.path.join(tmp_dir, 'v1.mp4') + v2 = os.path.join(tmp_dir, 'v2.mp4') + v3 = os.path.join(tmp_dir, 'v3.mp4') + out = os.path.join(tmp_dir, 'grid.mp4') + make_test_video(v1, duration=2, width=320, height=240) + make_test_video(v2, duration=2, width=640, height=480) + make_test_video(v3, duration=2, width=320, height=240) + sources = [GridSource(v1, 0), GridSource(v2, 0), GridSource(v3, 0)] + grid.composite(sources, 2.0, out, 1280, 720) + info = prober.probe_file(out) + assert info['width'] == 1280 + assert info['height'] == 720 + + +# --- PresenterLayout unit tests --- + +def test_presenter_main_only(ffmpeg, prober, tmp_dir): + """Presenter with only main source, no overlay.""" + presenter = PresenterLayout(ffmpeg) + v = os.path.join(tmp_dir, 'main.mp4') + out = os.path.join(tmp_dir, 'pres.mp4') + make_test_video(v, duration=3, with_audio=True) + main = GridSource(v, 0, has_audio=True) + presenter.composite(main, None, [], 3.0, out, 640, 360) + assert os.path.exists(out) + info = prober.probe_file(out) + assert info['width'] == 640 + assert info['height'] == 360 + assert info['has_audio'] + + +def test_presenter_with_pip(ffmpeg, prober, tmp_dir): + """Presenter with main + PIP overlay.""" + presenter = PresenterLayout(ffmpeg) + v_main = os.path.join(tmp_dir, 'main.mp4') + v_pip = os.path.join(tmp_dir, 'pip.mp4') + out = os.path.join(tmp_dir, 'pres.mp4') + make_test_video(v_main, duration=3, with_audio=True) + make_test_video(v_pip, duration=3, with_audio=True) + main = GridSource(v_main, 0, has_audio=True, is_admin=True) + overlay = GridSource(v_pip, 0, has_audio=True) + presenter.composite(main, overlay, [], 3.0, out, 640, 360) + assert os.path.exists(out) + info = prober.probe_file(out) + assert info['width'] == 640 + assert info['height'] == 360 + assert info['has_audio'] + + +def test_presenter_with_extra_audio(ffmpeg, prober, tmp_dir): + """Presenter with main + overlay + extra audio-only source.""" + presenter = PresenterLayout(ffmpeg) + v_main = os.path.join(tmp_dir, 'main.mp4') + v_pip = os.path.join(tmp_dir, 'pip.mp4') + v_extra = os.path.join(tmp_dir, 'extra.mp4') + out = os.path.join(tmp_dir, 'pres.mp4') + make_test_video(v_main, duration=3, with_audio=True) + make_test_video(v_pip, duration=3, with_audio=False) + make_test_video(v_extra, duration=3, with_audio=True) + main = GridSource(v_main, 0, has_audio=True, is_admin=True) + overlay = GridSource(v_pip, 0, has_audio=False) + extra = [GridSource(v_extra, 0, has_audio=True)] + presenter.composite(main, overlay, extra, 3.0, out, 640, 360) + assert os.path.exists(out) + info = prober.probe_file(out) + assert info['has_audio'] + + +def test_presenter_resolution_matches_target(ffmpeg, prober, tmp_dir): + """Presenter output must be exactly target_w x target_h.""" + presenter = PresenterLayout(ffmpeg) + # Source is 320x240 but target is 1280x720 + v_main = os.path.join(tmp_dir, 'main.mp4') + v_pip = os.path.join(tmp_dir, 'pip.mp4') + out = os.path.join(tmp_dir, 'pres.mp4') + make_test_video(v_main, duration=2, width=320, height=240, with_audio=True) + make_test_video(v_pip, duration=2, width=160, height=120, with_audio=True) + main = GridSource(v_main, 0, has_audio=True) + overlay = GridSource(v_pip, 0, has_audio=True) + presenter.composite(main, overlay, [], 2.0, out, 1280, 720) + info = prober.probe_file(out) + assert info['width'] == 1280 + assert info['height'] == 720 + + +# --- _build_audio_filter unit tests --- + +def test_build_audio_filter_no_audio(): + sources = [GridSource('a.mp4', 0, has_audio=False)] + filt, audio_map = _build_audio_filter(sources, 5.0) + assert filt == '' + assert audio_map == [] + + +def test_build_audio_filter_single(): + sources = [GridSource('a.mp4', 0, has_audio=True)] + filt, audio_map = _build_audio_filter(sources, 5.0) + assert '[aout]' in filt + assert audio_map == ['-map', '[aout]'] + + +def test_build_audio_filter_multi(): + sources = [ + GridSource('a.mp4', 0, has_audio=True), + GridSource('b.mp4', 0, has_audio=True), + ] + filt, audio_map = _build_audio_filter(sources, 5.0) + assert 'amix=inputs=2' in filt + assert audio_map == ['-map', '[aout]'] + + +def test_build_audio_filter_mixed(): + sources = [ + GridSource('a.mp4', 0, has_audio=True), + GridSource('b.mp4', 0, has_audio=False), + GridSource('c.mp4', 0, has_audio=True), + ] + filt, audio_map = _build_audio_filter(sources, 5.0) + assert 'amix=inputs=2' in filt + assert '[0:a]' in filt + assert '[2:a]' in filt + assert '[1:a]' not in filt + + +# --- GridCompositor facade backward compat --- + +def test_facade_composite(ffmpeg, tmp_dir): + """GridCompositor facade delegates to GridLayout.""" + comp = GridCompositor(ffmpeg) + v1 = os.path.join(tmp_dir, 'v1.mp4') + v2 = os.path.join(tmp_dir, 'v2.mp4') + out = os.path.join(tmp_dir, 'grid.mp4') + make_test_video(v1, duration=2) + make_test_video(v2, duration=2) + # Legacy tuple interface + comp.composite([(v1, 0), (v2, 0)], 2.0, out, 640, 360) + assert os.path.exists(out) + + +def test_facade_compute_layout(): + assert GridCompositor.compute_layout(4) == (2, 2) + + +def test_facade_even(): + assert GridCompositor.even(641) == 640 diff --git a/tests/test_prober.py b/tests/test_prober.py new file mode 100644 index 0000000..94f4722 --- /dev/null +++ b/tests/test_prober.py @@ -0,0 +1,81 @@ +import os +from tests.conftest import make_test_video, make_test_audio + + +def test_probe_streams_returns_streams(prober, tmp_dir): + path = os.path.join(tmp_dir, 'test.mp4') + make_test_video(path) + info = prober.probe_streams(path) + assert 'streams' in info + assert len(info['streams']) >= 1 + + +def test_has_video_true_for_video(prober, tmp_dir): + path = os.path.join(tmp_dir, 'vid.mp4') + make_test_video(path) + assert prober.has_video(path) is True + + +def test_has_video_false_for_audio(prober, tmp_dir): + path = os.path.join(tmp_dir, 'aud.m4a') + make_test_audio(path) + assert prober.has_video(path) is False + + +def test_has_audio_true(prober, tmp_dir): + path = os.path.join(tmp_dir, 'with.mp4') + make_test_video(path, with_audio=True) + assert prober.has_audio(path) is True + + +def test_has_audio_false(prober, tmp_dir): + path = os.path.join(tmp_dir, 'without.mp4') + make_test_video(path, with_audio=False) + assert prober.has_audio(path) is False + + +def test_get_duration(prober, tmp_dir): + path = os.path.join(tmp_dir, 'test.mp4') + make_test_video(path, duration=5) + dur = prober.get_duration(path) + assert 4.5 < dur < 5.5 + + +def test_get_video_params(prober, tmp_dir): + path = os.path.join(tmp_dir, 'test.mp4') + make_test_video(path, width=640, height=480) + w, h, pf = prober.get_video_params(path) + assert w == 640 + assert h == 480 + + +def test_probe_file(prober, tmp_dir): + path = os.path.join(tmp_dir, 'test.mp4') + make_test_video(path, duration=3, width=320, height=240) + info = prober.probe_file(path) + assert info['valid'] is True + assert info['has_video'] is True + assert info['has_audio'] is True + assert info['width'] == 320 + assert info['height'] == 240 + assert 2.5 < info['duration'] < 3.5 + + +def test_probe_file_invalid(prober, tmp_dir): + path = os.path.join(tmp_dir, 'bad.mp4') + with open(path, 'w') as f: + f.write('not a video') + info = prober.probe_file(path) + assert info['valid'] is False + + +def test_probe_all_files(prober, tmp_dir): + p1 = os.path.join(tmp_dir, 'v1.mp4') + p2 = os.path.join(tmp_dir, 'v2.mp4') + make_test_video(p1, duration=2) + make_test_video(p2, duration=3) + items = [(p1, 0.0), (p2, 5.0)] + results = prober.probe_all_files(items) + assert len(results) == 2 + assert results[0]['start_time'] == 0.0 + assert results[1]['start_time'] == 5.0 diff --git a/tests/test_processor.py b/tests/test_processor.py new file mode 100644 index 0000000..45f187b --- /dev/null +++ b/tests/test_processor.py @@ -0,0 +1,39 @@ +from mtslinker.processor import VideoProcessor +from mtslinker.ffmpeg import FFmpegRunner +from mtslinker.prober import MediaProber +from mtslinker.segments import SegmentBuilder +from mtslinker.grid import GridCompositor +from mtslinker.slides import SlideCompositor +from mtslinker.audio import AudioMerger + + +def test_processor_composes_all_classes(): + p = VideoProcessor() + assert p.ffmpeg is not None + assert p.prober is not None + assert p.segments is not None + assert p.grid is not None + assert p.slides is not None + assert p.audio is not None + + +def test_segments_uses_injected_ffmpeg(): + p = VideoProcessor() + assert p.segments.ffmpeg is p.ffmpeg + assert p.segments.prober is p.prober + + +def test_grid_uses_injected_ffmpeg(): + p = VideoProcessor() + assert p.grid.ffmpeg is p.ffmpeg + + +def test_slides_uses_injected_ffmpeg(): + p = VideoProcessor() + assert p.slides.ffmpeg is p.ffmpeg + + +def test_audio_uses_injected_deps(): + p = VideoProcessor() + assert p.audio.ffmpeg is p.ffmpeg + assert p.audio.prober is p.prober diff --git a/tests/test_segments.py b/tests/test_segments.py new file mode 100644 index 0000000..84d8836 --- /dev/null +++ b/tests/test_segments.py @@ -0,0 +1,158 @@ +import os +import subprocess + +import pytest + +from tests.conftest import make_test_video +from mtslinker.segments import SegmentBuilder + + +class _SpyFFmpeg: + """Captures the last ffmpeg command instead of running it.""" + + def __init__(self): + self.cmd = None + + def run(self, cmd, description='ffmpeg'): + self.cmd = cmd + + def get_video_encoder_fast(self): + return ['-c:v', 'libx264', '-preset', 'ultrafast', '-crf', '23'] + + +class _FakeProber: + """No audio in input, fixed duration — exercises the silent-audio path.""" + + def __init__(self, duration): + self._duration = duration + + def probe_streams(self, path): + return {'streams': [{'codec_type': 'video'}]} + + def get_duration(self, path): + return self._duration + + +def _flag_index(cmd, flag): + return cmd.index(flag) + + +def test_ensure_audio_guards_against_interleave_oom(): + """The silent-audio mux must not OOM via the muxer interleave buffer. + + Regression guard for av_interleaved_write_frame "Cannot allocate + memory": with -c:v copy + a lazily-generated anullsrc, ffmpeg buffers + the whole copied video in RAM unless we (a) cap the interleave buffer + and (b) make the silent input finite. A small test clip never triggers + the real OOM, so we assert the *command* keeps both safeguards. + """ + spy = _SpyFFmpeg() + builder = SegmentBuilder(spy, _FakeProber(duration=1234.0)) + builder.ensure_audio('in.mp4', 'out.mp4') + cmd = spy.cmd + + # (a) muxer must not buffer to interleave + assert '-max_interleave_delta' in cmd + assert cmd[cmd.index('-max_interleave_delta') + 1] == '0' + + # (b) anullsrc must be a finite input: -t before its -i, not after + anull = cmd.index('anullsrc=r=44100:cl=stereo') + t_idx = cmd.index('-t') + assert t_idx < anull, '-t must bound the anullsrc *input*, not the output' + assert cmd[t_idx + 1] == '1234.0' + + # correctness not regressed: video still stream-copied + assert cmd[cmd.index('-c:v') + 1] == 'copy' + + +def test_ensure_audio_rejects_zero_duration_input(): + """Empty/corrupt input (duration <= 0) must fail loudly, not run ffmpeg. + + Root cause of the production OOM: normalize() seeked past end-of-source + and produced a 262-byte, duration-0 file; feeding it to ensure_audio ran + an unbounded anullsrc + -c:v copy mux that buffered silent audio forever + until "av_interleaved_write_frame: Cannot allocate memory". ensure_audio + must refuse a non-positive-duration input (raising the same + CalledProcessError the callers' black-gap fallback catches) and must + never invoke ffmpeg in that case. + """ + spy = _SpyFFmpeg() + builder = SegmentBuilder(spy, _FakeProber(duration=0.0)) + with pytest.raises(subprocess.CalledProcessError): + builder.ensure_audio('in.mp4', 'out.mp4') + assert spy.cmd is None, 'ffmpeg must not run on a zero-duration input' + + +def test_normalize_fails_loudly_on_empty_output(): + """Seek past end-of-source: ffmpeg exits 0 with an empty container. + + normalize() must detect the zero-duration output and raise + CalledProcessError so the processor falls back to a black gap instead + of handing an empty file to ensure_audio (the OOM trigger). + """ + spy = _SpyFFmpeg() + builder = SegmentBuilder(spy, _FakeProber(duration=0.0)) + with pytest.raises(subprocess.CalledProcessError): + builder.normalize('in.mp4', 'out.mp4', 640, 360, 'yuv420p', + max_duration=90.0, seek=104.0) + # ffmpeg *was* attempted (the emptiness is only knowable post-run)... + assert spy.cmd is not None + # ...and it really was the normalize seek that ran. + assert spy.cmd[spy.cmd.index('-ss') + 1] == '104.0' + + +def test_normalize_succeeds_when_output_nonempty(): + """A normal (non-empty) normalize must still return the output path.""" + spy = _SpyFFmpeg() + builder = SegmentBuilder(spy, _FakeProber(duration=90.0)) + result = builder.normalize('in.mp4', 'out.mp4', 640, 360, 'yuv420p', + max_duration=90.0) + assert result == 'out.mp4' + + +def test_generate_black(segments, tmp_dir): + path = os.path.join(tmp_dir, 'black.mp4') + segments.generate_black(path, 2.0, 320, 240) + assert os.path.exists(path) + assert os.path.getsize(path) > 0 + + +def test_ensure_audio_keeps_existing(segments, tmp_dir): + vid = os.path.join(tmp_dir, 'with.mp4') + out = os.path.join(tmp_dir, 'out.mp4') + make_test_video(vid, with_audio=True) + result = segments.ensure_audio(vid, out) + assert result == vid + + +def test_ensure_audio_adds_silent(segments, tmp_dir): + from mtslinker.prober import MediaProber + vid = os.path.join(tmp_dir, 'without.mp4') + out = os.path.join(tmp_dir, 'out.mp4') + make_test_video(vid, with_audio=False, duration=5) + result = segments.ensure_audio(vid, out) + assert result == out + assert os.path.exists(out) + prober = MediaProber() + # Silent audio stream must actually be present... + assert prober.has_audio(out) + # ...and the -t bound must not truncate the video. + assert abs(prober.get_duration(out) - prober.get_duration(vid)) < 0.5 + + +def test_normalize(segments, tmp_dir): + vid = os.path.join(tmp_dir, 'input.mp4') + out = os.path.join(tmp_dir, 'norm.mp4') + make_test_video(vid) + segments.normalize(vid, out, 640, 360, 'yuv420p') + assert os.path.exists(out) + + +def test_normalize_with_max_duration(segments, tmp_dir): + vid = os.path.join(tmp_dir, 'input.mp4') + out = os.path.join(tmp_dir, 'norm.mp4') + make_test_video(vid, duration=5) + segments.normalize(vid, out, 640, 360, 'yuv420p', max_duration=2) + from mtslinker.prober import MediaProber + dur = MediaProber().get_duration(out) + assert dur < 3.0 diff --git a/tests/test_slides.py b/tests/test_slides.py new file mode 100644 index 0000000..8b0b652 --- /dev/null +++ b/tests/test_slides.py @@ -0,0 +1,178 @@ +import os +import subprocess + +import pytest + +from mtslinker.slides import SlideCompositor + + +class _FakeFFmpeg: + """Composition substitute for FFmpegRunner used in SlideCompositor tests. + + Records every (cmd, description) call. To make the next call matching + a description substring fail with a specific returncode, register it + via fail_with() before invoking the code under test. + """ + + def __init__(self): + self.calls = [] + self._pending_failures = [] + self.nvenc_available = True + self.cuda_overlay_available = False + + def get_video_encoder(self): + return ['-c:v', 'h264_nvenc', '-preset', 'p4', '-cq', '23'] + + def get_video_encoder_fast(self): + return ['-c:v', 'h264_nvenc', '-preset', 'p1', '-cq', '28'] + + def get_video_encoder_cpu(self): + return ['-c:v', 'libx264', '-preset', 'fast', '-crf', '23'] + + def fail_with(self, description_substring, returncode): + self._pending_failures.append((description_substring, returncode)) + + def run(self, cmd, description='ffmpeg'): + self.calls.append((list(cmd), description)) + for i, (sub, rc) in enumerate(self._pending_failures): + if sub == description: + self._pending_failures.pop(i) + raise subprocess.CalledProcessError(rc, cmd[0], '', '') + out = cmd[-1] + if isinstance(out, str) and not out.startswith('-'): + try: + with open(out, 'wb') as f: + f.write(b'\x00') + except OSError: + pass + + +class _StubSlideTrackCompositor(SlideCompositor): + """SlideCompositor with the slide-track build stubbed out. + + The OOM-retry logic under test lives in composite()'s chunked branch; + _build_slide_track does many small ffmpeg calls that would clutter the + recorded call list and tell us nothing about retry behavior. Touching + a single empty file is enough — the chunked composite reads the path, + not the contents. + """ + + def _build_slide_track(self, slide_events, tmp_dir, total_duration): + path = os.path.join(tmp_dir, 'slide_track.mp4') + open(path, 'wb').close() + return path + + +def _enc_codec(cmd): + return cmd[cmd.index('-c:v') + 1] + + +def _composite_chunk_calls(calls): + # Match the per-chunk encode calls only; the final 'concat composite + # chunks' concat-demuxer step also contains "composite chunk" as a + # substring and would otherwise pollute the assertions. + return [(c, d) for c, d in calls if d.startswith('composite chunk')] + + +def _slide_events(): + return [{'time': 0, 'local_path': 'x.jpg', 'slide_number': 1}] + + +def test_composite_chunk_retries_on_oom_kill_with_cpu_encoder(tmp_path): + """exit -9 SIGKILL on NVENC must trigger a CPU retry of that chunk. + + Reproduces the production OOM-kill on `composite chunk 2/8` (URL + ...record-new/2413910825): the slide compositor's NVENC pipeline was + killed by the kernel OOM killer mid-chunk. The correct response is to + retry that chunk with libx264 and stay on CPU for the rest of the + video so the next chunk does not pay the kill cost again. + """ + ffmpeg = _FakeFFmpeg() + ffmpeg.fail_with('composite chunk 2/3', -9) + compositor = _StubSlideTrackCompositor(ffmpeg) + + video = os.path.join(str(tmp_path), 'video.mp4') + open(video, 'wb').close() + out = os.path.join(str(tmp_path), 'final.mp4') + compositor.composite(video, _slide_events(), out, str(tmp_path), 5400.0) + + chunks = _composite_chunk_calls(ffmpeg.calls) + descs = [d for _, d in chunks] + assert descs == [ + 'composite chunk 1/3', + 'composite chunk 2/3', + 'composite chunk 2/3 (cpu retry)', + 'composite chunk 3/3 (cpu)', + ], descs + assert _enc_codec(chunks[0][0]) == 'h264_nvenc' + assert _enc_codec(chunks[1][0]) == 'h264_nvenc' + assert _enc_codec(chunks[2][0]) == 'libx264' + assert _enc_codec(chunks[3][0]) == 'libx264' + + +def test_composite_chunk_does_not_retry_on_non_oom_failure(tmp_path): + """A regular nonzero exit must propagate — only SIGKILL means OOM. + + Falling back to CPU on every ffmpeg error would hide real bugs (bad + inputs, codec errors, etc.). Only the kernel OOM killer signals + memory pressure, so only that should trigger the silent retry. + """ + ffmpeg = _FakeFFmpeg() + ffmpeg.fail_with('composite chunk 1/3', 1) + compositor = _StubSlideTrackCompositor(ffmpeg) + + video = os.path.join(str(tmp_path), 'video.mp4') + open(video, 'wb').close() + out = os.path.join(str(tmp_path), 'final.mp4') + with pytest.raises(subprocess.CalledProcessError): + compositor.composite(video, _slide_events(), out, str(tmp_path), 5400.0) + + chunks = _composite_chunk_calls(ffmpeg.calls) + assert len(chunks) == 1 + assert chunks[0][1] == 'composite chunk 1/3' + + +def test_composite_chunk_treats_exit_137_as_oom_kill(tmp_path): + """Some shells surface SIGKILL as 128+9=137 instead of Python's -9. + + Both must trigger the same CPU retry path so the fallback is not + sensitive to the kernel/wrapper that reaped ffmpeg. + """ + ffmpeg = _FakeFFmpeg() + ffmpeg.fail_with('composite chunk 1/3', 137) + compositor = _StubSlideTrackCompositor(ffmpeg) + + video = os.path.join(str(tmp_path), 'video.mp4') + open(video, 'wb').close() + out = os.path.join(str(tmp_path), 'final.mp4') + compositor.composite(video, _slide_events(), out, str(tmp_path), 5400.0) + + chunks = _composite_chunk_calls(ffmpeg.calls) + descs = [d for _, d in chunks] + assert descs == [ + 'composite chunk 1/3', + 'composite chunk 1/3 (cpu retry)', + 'composite chunk 2/3 (cpu)', + 'composite chunk 3/3 (cpu)', + ], descs + + +def test_composite_chunk_uses_nvenc_when_nothing_fails(tmp_path): + """The happy path must keep using NVENC — the CPU fallback is only + engaged after an actual OOM-kill, never preemptively.""" + ffmpeg = _FakeFFmpeg() + compositor = _StubSlideTrackCompositor(ffmpeg) + + video = os.path.join(str(tmp_path), 'video.mp4') + open(video, 'wb').close() + out = os.path.join(str(tmp_path), 'final.mp4') + compositor.composite(video, _slide_events(), out, str(tmp_path), 5400.0) + + chunks = _composite_chunk_calls(ffmpeg.calls) + assert [d for _, d in chunks] == [ + 'composite chunk 1/3', + 'composite chunk 2/3', + 'composite chunk 3/3', + ] + for cmd, _ in chunks: + assert _enc_codec(cmd) == 'h264_nvenc' diff --git a/tests/test_timeline.py b/tests/test_timeline.py new file mode 100644 index 0000000..4956428 --- /dev/null +++ b/tests/test_timeline.py @@ -0,0 +1,217 @@ +from mtslinker.timeline import StreamTimeline, MediaSession, TimeWindow, DownloadChunk + + +def _make_json(events, duration=100.0): + return {'duration': duration, 'eventLogs': events} + + +def _ms_add(ms_id, url, time, conf_id=None, screenshare_id=None): + data = {'id': ms_id, 'url': url} + stream = {} + if conf_id: + stream['conference'] = {'id': conf_id} + if screenshare_id: + stream['screensharing'] = {'id': screenshare_id} + if stream: + data['stream'] = stream + return {'module': 'mediasession.add', 'relativeTime': time, 'data': data} + + +def _ms_update(ms_id, duration): + return {'module': 'mediasession.update', 'relativeTime': 0, + 'data': {'id': ms_id, 'duration': duration}} + + +def _conf_add(conf_id, user_id, has_audio=True, has_video=True): + return {'module': 'conference.add', 'relativeTime': 0, + 'data': {'id': conf_id, 'user': {'id': user_id}, + 'hasAudio': has_audio, 'hasVideo': has_video}} + + +def test_single_session(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 100.0), + ]) + tl = StreamTimeline() + windows = tl.build(json_data) + assert len(windows) == 1 + assert windows[0].start_time == 0.0 + assert abs(windows[0].duration - 100.0) < 0.2 + assert len(windows[0].streams) == 1 + assert windows[0].streams[0].url == 'http://a.mp4' + + +def test_two_overlapping_sessions(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 60.0), + _ms_add('ms2', 'http://b.mp4', 10.0), + _ms_update('ms2', 50.0), + ], duration=60.0) + tl = StreamTimeline() + windows = tl.build(json_data) + # Windows: [0-10] ms1 only, [10-60] ms1+ms2 + assert len(windows) == 2 + assert len(windows[0].streams) == 1 + assert len(windows[1].streams) == 2 + + +def test_sequential_sessions(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 30.0), + _ms_add('ms2', 'http://b.mp4', 30.0), + _ms_update('ms2', 30.0), + ], duration=60.0) + tl = StreamTimeline() + windows = tl.build(json_data) + assert len(windows) == 2 + assert len(windows[0].streams) == 1 + assert windows[0].streams[0].url == 'http://a.mp4' + assert len(windows[1].streams) == 1 + assert windows[1].streams[0].url == 'http://b.mp4' + + +def test_gap_between_sessions(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 20.0), + _ms_add('ms2', 'http://b.mp4', 40.0), + _ms_update('ms2', 20.0), + ], duration=60.0) + tl = StreamTimeline() + windows = tl.build(json_data) + # [0-20] ms1, [20-40] empty, [40-60] ms2 + assert len(windows) == 3 + assert len(windows[0].streams) == 1 + assert len(windows[1].streams) == 0 + assert len(windows[2].streams) == 1 + + +def test_conference_audio_video_flags(): + json_data = _make_json([ + _conf_add('c1', 'u1', has_audio=True, has_video=False), + _ms_add('ms1', 'http://a.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + ], duration=50.0) + tl = StreamTimeline() + windows = tl.build(json_data) + assert len(windows) == 1 + s = windows[0].streams[0] + assert s.has_audio is True + assert s.has_video is False + + +def test_admin_detection(): + json_data = _make_json([ + {'module': 'userlist.add', 'relativeTime': 0, + 'data': {'role': 'ADMIN', 'user': {'id': 'u1'}}}, + _conf_add('c1', 'u1'), + _ms_add('ms1', 'http://a.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + assert 'c1' in tl.admin_conf_ids + + +def test_get_download_chunks(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + _ms_add('ms2', 'http://b.mp4', 10.0, conf_id='c2'), + _ms_update('ms2', 40.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + dl = tl.get_download_chunks() + assert len(dl) == 2 + assert dl[0].url == 'http://a.mp4' + assert dl[0].start_time == 0.0 + assert dl[1].url == 'http://b.mp4' + + +def test_map_downloaded_files(): + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + tl.map_downloaded_files([('/tmp/a.mp4', 0.0, 'c1', False, 50.0)]) + assert tl.sessions['ms1'].file_path == '/tmp/a.mp4' + + +def test_three_webcam_overlap(): + """Three webcams with staggered starts — should produce multiple windows.""" + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 100.0), + _ms_add('ms2', 'http://b.mp4', 20.0), + _ms_update('ms2', 80.0), + _ms_add('ms3', 'http://c.mp4', 40.0), + _ms_update('ms3', 60.0), + ], duration=100.0) + tl = StreamTimeline() + windows = tl.build(json_data) + # [0-20] 1 stream, [20-40] 2 streams, [40-100] 3 streams + assert len(windows) == 3 + assert len(windows[0].streams) == 1 + assert len(windows[1].streams) == 2 + assert len(windows[2].streams) == 3 + + +def test_empty_events(): + json_data = _make_json([]) + tl = StreamTimeline() + windows = tl.build(json_data) + assert windows == [] + + +def test_fallback_url_capture(): + """URLs not in mediasession.add should be captured as fallback sessions.""" + json_data = _make_json([ + {'module': 'someother.event', 'relativeTime': 5.0, + 'data': {'url': 'http://fallback.mp4'}}, + ], duration=50.0) + tl = StreamTimeline() + windows = tl.build(json_data) + assert len(tl.sessions) == 1 + assert any('fallback' in sid for sid in tl.sessions) + + +def test_screenshare_detection(): + """mediasession with stream.screensharing should be flagged.""" + json_data = _make_json([ + _ms_add('ms1', 'http://webcam.mp4', 0.0, conf_id='c1'), + _ms_update('ms1', 50.0), + _ms_add('ms2', 'http://screen.mp4', 10.0, screenshare_id=99), + _ms_update('ms2', 40.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + assert tl.sessions['ms1'].is_screenshare is False + assert tl.sessions['ms2'].is_screenshare is True + + +def test_screenshare_not_detected_without_id(): + """Empty screensharing dict should not flag as screenshare.""" + json_data = _make_json([ + _ms_add('ms1', 'http://a.mp4', 0.0), + _ms_update('ms1', 50.0), + ], duration=50.0) + tl = StreamTimeline() + tl.build(json_data) + assert tl.sessions['ms1'].is_screenshare is False + + +def test_time_window_properties(): + w = TimeWindow(start_time=10.0, duration=5.0, streams=[ + MediaSession(id='1', url='a', start_time=0, has_video=True, has_audio=False), + MediaSession(id='2', url='b', start_time=0, has_video=False, has_audio=True), + ]) + assert w.end_time == 15.0 + assert w.has_video is True + assert w.has_audio is True + assert w.stream_count == 2