From 8e0972682f3d7eba119b0e8e5e2ef12e6357eacb Mon Sep 17 00:00:00 2001 From: DorkSoul <71715506+DorkSoul@users.noreply.github.com> Date: Tue, 17 Mar 2026 13:51:22 +0000 Subject: [PATCH 1/7] rewrite and fixes --- .github/workflows/docker-publish.yml | 4 +- app/app.py | 3 + app/routes/download_routes.py | 10 + app/routes/events_routes.py | 97 +++++ app/scheduler.py | 264 +++++------- app/services/browser_service.py | 11 +- app/services/download_service.py | 616 ++++++++++++--------------- app/static/js/downloads-browser.js | 157 ++++--- app/static/js/state.js | 1 + 9 files changed, 588 insertions(+), 575 deletions(-) create mode 100644 app/routes/events_routes.py diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 51b66dd..2e65f96 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - testing workflow_dispatch: env: @@ -43,7 +44,8 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=sha - type=raw,value=latest,enable={{is_default_branch}} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=testing,enable=${{ github.ref == 'refs/heads/testing' }} - name: Build and push Docker image uses: docker/build-push-action@v5 diff --git a/app/app.py b/app/app.py index 166c7a8..8ca2c55 100644 --- a/app/app.py +++ b/app/app.py @@ -4,6 +4,7 @@ from app.scheduler import Scheduler from app.routes import init_browser_routes, init_download_routes from app.routes.scheduler_routes import init_scheduler_routes +from app.routes.events_routes import init_events_routes def create_app(): @@ -32,11 +33,13 @@ def create_app(): browser_bp = init_browser_routes(browser_service, download_service, config) download_bp = init_download_routes(download_service, config.DOWNLOAD_DIR, scheduler) scheduler_bp = init_scheduler_routes(scheduler) + events_bp = init_events_routes(browser_service, download_service) # Register blueprints flask_app.register_blueprint(browser_bp) flask_app.register_blueprint(download_bp) flask_app.register_blueprint(scheduler_bp) + flask_app.register_blueprint(events_bp) # Main route @flask_app.route('/') diff --git a/app/routes/download_routes.py b/app/routes/download_routes.py index 66e9f21..daa8f1d 100644 --- a/app/routes/download_routes.py +++ b/app/routes/download_routes.py @@ -278,4 +278,14 @@ def delete_download(filename): logger.error(f"Delete download error: {e}") return jsonify({'error': str(e)}), 500 + @download_bp.route('/history', methods=['GET']) + def download_history(): + """Return the persisted download history log.""" + try: + history = download_service.get_history() + return jsonify({'history': history}) + except Exception as e: + logger.error(f"History error: {e}") + return jsonify({'error': str(e)}), 500 + return download_bp diff --git a/app/routes/events_routes.py b/app/routes/events_routes.py new file mode 100644 index 0000000..7562e07 --- /dev/null +++ b/app/routes/events_routes.py @@ -0,0 +1,97 @@ +import json +import time +import logging +from flask import Blueprint, Response, stream_with_context + +logger = logging.getLogger(__name__) + +events_bp = Blueprint('events', __name__, url_prefix='/api/events') + + +def init_events_routes(browser_service, download_service): + """Initialize SSE event-stream routes.""" + + @events_bp.route('/browser/') + def browser_events(browser_id): + """ + SSE stream for a specific browser/download session. + + Replaces polling on /api/browser/status/. + Closes automatically when the browser/download is no longer running. + """ + def generate(): + while True: + try: + status = browser_service.get_browser_status(browser_id) + + if status is None: + # Check direct download status as fallback + with download_service._queue_lock: + direct = download_service.direct_download_status.get(browser_id) + if direct: + status = dict(direct) + else: + # Browser gone — send a final closed event and stop + yield f"data: {json.dumps({'is_running': False, 'closed': True})}\n\n" + return + + # Attach download info if available + download_info = download_service.get_download_status(browser_id) + if download_info: + status['download'] = download_info + + yield f"data: {json.dumps(status)}\n\n" + + # Stop streaming once the browser/download has finished + if not status.get('is_running', True): + return + + except Exception as e: + logger.error(f"SSE error for browser {browser_id}: {e}") + yield f"data: {json.dumps({'error': str(e)})}\n\n" + + time.sleep(2) + + return Response( + stream_with_context(generate()), + mimetype='text/event-stream', + headers={ + 'Cache-Control': 'no-cache', + 'X-Accel-Buffering': 'no', + 'Connection': 'keep-alive', + }, + ) + + @events_bp.route('/active') + def active_events(): + """ + SSE stream for the active downloads list. + + Streams a compact (no thumbnail data) snapshot every 3 seconds. + Thumbnails are still fetched on demand via /api/browser/status/. + """ + def generate(): + while True: + try: + active = download_service.get_active_downloads() + # Strip bulky thumbnail data to keep the SSE stream lightweight + compact = [ + {k: v for k, v in d.items() if k != 'thumbnail'} + for d in active + ] + yield f"data: {json.dumps({'active_downloads': compact})}\n\n" + except Exception as e: + logger.error(f"SSE active events error: {e}") + time.sleep(3) + + return Response( + stream_with_context(generate()), + mimetype='text/event-stream', + headers={ + 'Cache-Control': 'no-cache', + 'X-Accel-Buffering': 'no', + 'Connection': 'keep-alive', + }, + ) + + return events_bp diff --git a/app/scheduler.py b/app/scheduler.py index b7125b8..12e414e 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -1,6 +1,7 @@ import os import time import json +import uuid import logging import threading import random @@ -18,6 +19,7 @@ class Scheduler: - Manual stop detection and handling - Specific download tracking for multi-stream support - Thread-safe schedule management with JSON persistence + - Lazy disk writes (only when schedules are actually modified) """ def __init__(self, config, browser_service): @@ -27,7 +29,8 @@ def __init__(self, config, browser_service): self.running = False self.thread = None self.lock = threading.Lock() - + self._dirty = False # True when in-memory schedules differ from disk + self.load_schedules() def load_schedules(self): @@ -50,36 +53,42 @@ def load_schedules(self): self.schedules = [] def save_schedules(self): - """Save schedules to disk""" + """Save schedules to disk (only if dirty)""" + if not self._dirty: + return try: with open(self.config.SCHEDULES_FILE, 'w') as f: json.dump(self.schedules, f, indent=2) + self._dirty = False except Exception as e: logger.error(f"Error saving schedules: {e}") + def _mark_dirty(self): + """Mark schedules as needing a save.""" + self._dirty = True + def add_schedule(self, url, start_time, end_time, repeat=False, daily=False, name=None, resolution='1080p', framerate='any', format='mp4'): """Add a new schedule""" with self.lock: schedule = { - 'id': str(int(time.time() * 1000)), + 'id': uuid.uuid4().hex, 'url': url, - 'name': name, # Optional name for filename prefix (not defaulting to URL) + 'name': name, 'resolution': resolution, 'framerate': framerate, 'format': format, - 'start_time': start_time, # ISO format string or HH:MM for daily - 'end_time': end_time, # ISO format string or HH:MM for daily + 'start_time': start_time, + 'end_time': end_time, 'repeat': repeat, - 'daily': daily, # If true, start_time and end_time are HH:MM format - 'status': 'pending', # pending, active, completed, download_started + 'daily': daily, + 'status': 'pending', 'next_check': None, 'last_check': None, 'created_at': datetime.now().isoformat() } - # Initialize next check self._update_next_check(schedule) - self.schedules.append(schedule) + self._mark_dirty() self.save_schedules() return schedule @@ -87,6 +96,7 @@ def remove_schedule(self, schedule_id): """Remove a schedule""" with self.lock: self.schedules = [s for s in self.schedules if s['id'] != schedule_id] + self._mark_dirty() self.save_schedules() return True @@ -95,9 +105,8 @@ def update_schedule(self, schedule_id, url, start_time, end_time, repeat=False, with self.lock: for schedule in self.schedules: if schedule['id'] == schedule_id: - # Update fields schedule['url'] = url - schedule['name'] = name # Optional name, not defaulting to URL + schedule['name'] = name schedule['start_time'] = start_time schedule['end_time'] = end_time schedule['repeat'] = repeat @@ -105,13 +114,9 @@ def update_schedule(self, schedule_id, url, start_time, end_time, repeat=False, schedule['resolution'] = resolution schedule['framerate'] = framerate schedule['format'] = format - - # Reset status if times changed schedule['status'] = 'pending' - - # Update next check time self._update_next_check(schedule) - + self._mark_dirty() self.save_schedules() logger.info(f"Updated schedule {schedule_id}") return schedule @@ -124,18 +129,10 @@ def move_to_next_slot(self, browser_id): When user manually stops a download, the schedule is reset to 'pending' and moved to the next occurrence (next day for daily, next week for weekly). - - Args: - browser_id: The browser_id of the stopped download - - Returns: - bool: True if schedule was found and moved, False otherwise """ - # Check if this is a scheduled download (format: sched_{schedule_id}_{timestamp}) if not browser_id.startswith('sched_'): return False - # Extract schedule_id from browser_id parts = browser_id.split('_') if len(parts) < 2: return False @@ -145,37 +142,29 @@ def move_to_next_slot(self, browser_id): with self.lock: for schedule in self.schedules: if schedule['id'] == schedule_id: - # Only process if this is the active browser_id if schedule.get('active_browser_id') == browser_id: - # Reset schedule state schedule['status'] = 'pending' schedule['active_browser_id'] = None schedule['manual_stop'] = False - # Move to next time slot if schedule.get('daily'): - # Daily schedule - explicitly move to tomorrow's start time logger.info(f"Moving daily schedule {schedule_id} to next day") start_time_str = schedule['start_time'] start_hour, start_min = map(int, start_time_str.split(':')) - - # Calculate tomorrow's start time now = datetime.now() tomorrow = now.date() + timedelta(days=1) next_start = datetime.combine(tomorrow, datetime.min.time().replace(hour=start_hour, minute=start_min)) - schedule['next_check'] = next_start.isoformat() logger.info(f"Daily schedule {schedule_id} moved to {next_start}") elif schedule.get('repeat'): - # Weekly recurring - move to next week logger.info(f"Moving weekly schedule {schedule_id} to next week") self._reschedule_next_week(schedule) else: - # One-time schedule - mark as completed logger.info(f"One-time schedule {schedule_id} stopped, marking as completed") schedule['status'] = 'completed' self._update_next_check(schedule) + self._mark_dirty() self.save_schedules() logger.info(f"Schedule {schedule_id} moved to next time slot (browser_id: {browser_id})") return True @@ -184,13 +173,11 @@ def move_to_next_slot(self, browser_id): def get_schedules(self): """Get all schedules, sorted by next_check time""" - # Sort schedules by next_check time (soonest first) - # Schedules without next_check go to the end sorted_schedules = sorted( self.schedules, key=lambda s: ( - s.get('next_check') is None, # False (0) for schedules with next_check, True (1) for those without - s.get('next_check') or '' # Sort by next_check if it exists + s.get('next_check') is None, + s.get('next_check') or '' ) ) return sorted_schedules @@ -199,23 +186,20 @@ def refresh_all_schedule_times(self): """ Reset all schedules as if they were just created. - This clears all manual stops, resets statuses to pending, - and recalculates next_check times. If a schedule's time window - is currently active, it will start checking for streams again. + Clears all manual stops, resets statuses to pending, + and recalculates next_check times. """ with self.lock: count = 0 for schedule in self.schedules: - # Reset schedule state completely schedule['status'] = 'pending' schedule['active_browser_id'] = None schedule['manual_stop'] = False schedule['last_check'] = None - - # Recalculate next_check time self._update_next_check(schedule) count += 1 + self._mark_dirty() self.save_schedules() logger.info(f"Reset {count} schedules to fresh state") return count @@ -224,7 +208,7 @@ def start(self): """Start the scheduler loop""" if self.running: return - + self.running = True self.thread = threading.Thread(target=self._run_loop, daemon=True) self.thread.start() @@ -245,11 +229,9 @@ def _run_loop(self): self._check_schedules() except Exception as e: logger.error(f"Scheduler loop error: {e}") - - # Sleep for a bit before next iteration (e.g., 30 seconds) - # We don't need super high precision + for _ in range(30): - if not self.running: + if not self.running: break time.sleep(1) @@ -260,53 +242,44 @@ def _check_schedules(self): with self.lock: for schedule in self.schedules: if schedule['status'] == 'completed' and not schedule.get('daily'): - # Skip completed non-daily schedules + continue + + # Skip schedules that are already being checked by a running thread + if schedule.get('status') == 'checking': continue try: if schedule.get('daily'): - # Daily schedule - handle time-based windows self._check_daily_schedule(schedule, now) else: - # Regular schedule - handle datetime-based windows start_dt = datetime.fromisoformat(schedule['start_time']) end_dt = datetime.fromisoformat(schedule['end_time']) - # Check if window passed if now > end_dt: if schedule['repeat']: - # Move to next week self._reschedule_next_week(schedule) else: if schedule['status'] != 'download_started': schedule['status'] = 'completed' - # Clear flags when window ends schedule['active_browser_id'] = None schedule['manual_stop'] = False continue - # Check if currently active window if start_dt <= now <= end_dt: if schedule['status'] == 'download_started': - # Check if the specific download is still active active_browser_id = schedule.get('active_browser_id') if self._is_download_active(active_browser_id): - # Download still running, keep status and skip check continue else: - # Download crashed/failed automatically - resume checking logger.info(f"Download {active_browser_id} stopped for schedule {schedule['id']}, resuming stream checks") schedule['status'] = 'active' schedule['active_browser_id'] = None - # Will proceed to check stream below - # Ensure status is active, but only if next_check is within current window if schedule['status'] != 'active': next_check_val = schedule.get('next_check') if next_check_val: try: next_check_dt = datetime.fromisoformat(next_check_val) - # Only change to active if next_check is not beyond current window if next_check_dt <= end_dt: schedule['status'] = 'active' except (ValueError, TypeError): @@ -314,26 +287,22 @@ def _check_schedules(self): else: schedule['status'] = 'active' - # Check if it's time to check stream next_check = schedule.get('next_check') if not next_check or now >= datetime.fromisoformat(next_check): - # It's time! (when next_check time arrives) self._perform_check(schedule) elif now < start_dt: - schedule['status'] = 'pending' - # Ensure next_check is set (only if missing or in the past) - next_check = schedule.get('next_check') - if not next_check: - self._update_next_check(schedule) - else: - try: - next_check_dt = datetime.fromisoformat(next_check) - # Only recalculate if next_check is in the past - if next_check_dt < now: - self._update_next_check(schedule) - except (ValueError, TypeError): - self._update_next_check(schedule) + schedule['status'] = 'pending' + next_check = schedule.get('next_check') + if not next_check: + self._update_next_check(schedule) + else: + try: + next_check_dt = datetime.fromisoformat(next_check) + if next_check_dt < now: + self._update_next_check(schedule) + except (ValueError, TypeError): + self._update_next_check(schedule) except Exception as e: logger.error(f"Error processing schedule {schedule['id']}: {e}") @@ -346,66 +315,45 @@ def _check_daily_schedule(self, schedule, now): Handles: - Midnight-spanning windows (e.g., 23:00-01:00) - - Status transitions (pending -> active -> download_started) + - Status transitions (pending -> active -> checking -> download_started) - Auto-resume on download failures (unless manually stopped) - Proper reset when time window ends """ - # Parse the time strings (format: "HH:MM") start_time_str = schedule['start_time'] end_time_str = schedule['end_time'] - # Get today's date today = now.date() - - # Create datetime objects for today's window start_hour, start_min = map(int, start_time_str.split(':')) end_hour, end_min = map(int, end_time_str.split(':')) start_dt = datetime.combine(today, datetime.min.time().replace(hour=start_hour, minute=start_min)) end_dt = datetime.combine(today, datetime.min.time().replace(hour=end_hour, minute=end_min)) - # Detect if this is a midnight-spanning window (e.g., 23:00 - 01:00) spans_midnight = end_hour < start_hour or (end_hour == start_hour and end_min < start_min) if spans_midnight: - # For midnight-spanning windows, we need to check if we're in yesterday's window - # that extends into today, OR in today's window that extends into tomorrow - - # Check if we're in yesterday's window (before today's start time) if now.time() < datetime.min.time().replace(hour=start_hour, minute=start_min): - # We're in the early morning hours - check if yesterday's window extends to now yesterday = today - timedelta(days=1) start_dt = datetime.combine(yesterday, datetime.min.time().replace(hour=start_hour, minute=start_min)) end_dt = datetime.combine(today, datetime.min.time().replace(hour=end_hour, minute=end_min)) else: - # We're after start time today - window extends into tomorrow end_dt = end_dt + timedelta(days=1) - else: - # Normal same-day window - pass - # Check if we're currently in the active window if start_dt <= now <= end_dt: if schedule['status'] == 'download_started': - # Check if the specific download is still active active_browser_id = schedule.get('active_browser_id') if self._is_download_active(active_browser_id): - # Download still running, keep status and skip check return else: - # Download crashed/failed automatically - resume checking logger.info(f"Download {active_browser_id} stopped for schedule {schedule['id']}, resuming stream checks") schedule['status'] = 'active' schedule['active_browser_id'] = None - # Will proceed to check stream below - # Ensure status is active, but only if next_check is within current window if schedule['status'] != 'active': next_check_val = schedule.get('next_check') if next_check_val: try: next_check_dt = datetime.fromisoformat(next_check_val) - # Only change to active if next_check is not beyond current window if next_check_dt <= end_dt: schedule['status'] = 'active' except (ValueError, TypeError): @@ -413,36 +361,29 @@ def _check_daily_schedule(self, schedule, now): else: schedule['status'] = 'active' - # Check if it's time to check stream next_check = schedule.get('next_check') if not next_check or now >= datetime.fromisoformat(next_check): - # It's time! (when next_check time arrives) self._perform_check(schedule) elif now < start_dt: - # Window hasn't started yet schedule['status'] = 'pending' - # Ensure next_check is set (only if missing or in the past) next_check = schedule.get('next_check') if not next_check: self._update_next_check(schedule) else: try: next_check_dt = datetime.fromisoformat(next_check) - # Only recalculate if next_check is in the past if next_check_dt < now: self._update_next_check(schedule) except (ValueError, TypeError): self._update_next_check(schedule) else: - # Window has passed - always reset to pending for next day - if schedule['status'] in ['active', 'download_started']: - # Reset for next day + if schedule['status'] in ['active', 'download_started', 'checking']: schedule['status'] = 'pending' schedule['last_check'] = None - schedule['active_browser_id'] = None # Clear tracked browser_id - schedule['manual_stop'] = False # Clear manual stop flag for next time window + schedule['active_browser_id'] = None + schedule['manual_stop'] = False self._update_next_check(schedule) def _update_next_check(self, schedule): @@ -458,7 +399,6 @@ def _update_next_check(self, schedule): now = datetime.now() if schedule.get('daily'): - # Daily schedule - calculate based on time start_time_str = schedule['start_time'] end_time_str = schedule['end_time'] @@ -469,42 +409,28 @@ def _update_next_check(self, schedule): start_dt = datetime.combine(today, datetime.min.time().replace(hour=start_hour, minute=start_min)) end_dt = datetime.combine(today, datetime.min.time().replace(hour=end_hour, minute=end_min)) - # Detect if this is a midnight-spanning window (e.g., 23:00 - 01:00) spans_midnight = end_hour < start_hour or (end_hour == start_hour and end_min < start_min) if spans_midnight: - # For midnight-spanning windows, determine which window we're checking if now.time() < datetime.min.time().replace(hour=start_hour, minute=start_min): - # We're in the early morning hours - check if yesterday's window extends to now yesterday = today - timedelta(days=1) start_dt = datetime.combine(yesterday, datetime.min.time().replace(hour=start_hour, minute=start_min)) end_dt = datetime.combine(today, datetime.min.time().replace(hour=end_hour, minute=end_min)) else: - # We're after start time today - window extends into tomorrow end_dt = end_dt + timedelta(days=1) - # If window hasn't started yet, schedule check for start of window if now < start_dt: schedule['next_check'] = start_dt.isoformat() logger.debug(f"Schedule {schedule['id']}: next check set to window start: {start_dt}") - # If we're in the window, schedule random check in 5-8 minutes elif start_dt <= now <= end_dt: minutes = random.uniform(5, 8) - next_dt = now + timedelta(minutes=minutes) - # Make sure we don't schedule past the end of the window - if next_dt > end_dt: - next_dt = end_dt + next_dt = min(now + timedelta(minutes=minutes), end_dt) schedule['next_check'] = next_dt.isoformat() logger.debug(f"Schedule {schedule['id']}: next check in {minutes:.1f} mins: {next_dt}") - # If window has passed, schedule for next occurrence else: - # For midnight-spanning, if we're past end time but before start time, - # the next window is today (later). Otherwise it's tomorrow. if spans_midnight and now.time() < datetime.min.time().replace(hour=start_hour, minute=start_min): - # We're past yesterday's window end, next window is today next_start = datetime.combine(today, datetime.min.time().replace(hour=start_hour, minute=start_min)) else: - # Next window is tomorrow tomorrow = today + timedelta(days=1) next_start = datetime.combine(tomorrow, datetime.min.time().replace(hour=start_hour, minute=start_min)) @@ -512,24 +438,17 @@ def _update_next_check(self, schedule): logger.debug(f"Schedule {schedule['id']}: next check set to next window start: {next_start}") else: - # Regular schedule - calculate based on datetime start_dt = datetime.fromisoformat(schedule['start_time']) end_dt = datetime.fromisoformat(schedule['end_time']) - # If window hasn't started yet, schedule check for start of window if now < start_dt: schedule['next_check'] = start_dt.isoformat() logger.debug(f"Schedule {schedule['id']}: next check set to window start: {start_dt}") - # If we're in the window, schedule random check in 5-8 minutes elif start_dt <= now <= end_dt: minutes = random.uniform(5, 8) - next_dt = now + timedelta(minutes=minutes) - # Make sure we don't schedule past the end of the window - if next_dt > end_dt: - next_dt = end_dt + next_dt = min(now + timedelta(minutes=minutes), end_dt) schedule['next_check'] = next_dt.isoformat() logger.debug(f"Schedule {schedule['id']}: next check in {minutes:.1f} mins: {next_dt}") - # If window has passed, clear next_check (will be rescheduled) else: schedule['next_check'] = None logger.debug(f"Schedule {schedule['id']}: window passed, clearing next_check") @@ -545,26 +464,17 @@ def _reschedule_next_week(self, schedule): schedule['start_time'] = new_start.isoformat() schedule['end_time'] = new_end.isoformat() schedule['status'] = 'pending' - schedule['active_browser_id'] = None # Clear tracked browser_id - schedule['manual_stop'] = False # Clear manual stop flag for next time window + schedule['active_browser_id'] = None + schedule['manual_stop'] = False - # Update next_check to the new window start self._update_next_check(schedule) - logger.info(f"Rescheduled {schedule['id']} to next week: {new_start}") def _is_download_active(self, browser_id): """ Check if a specific download is still active. - A download is considered active if it exists in the download_queue - and doesn't have a 'completed_at' timestamp. - - Args: - browser_id: The specific browser_id to check - - Returns: - bool: True if the download is still active, False otherwise + Returns True if the download exists in the queue and has no completed_at. """ try: if not browser_id: @@ -574,7 +484,6 @@ def _is_download_active(self, browser_id): if browser_id in download_queue: download_info = download_queue[browser_id] - # Check if download is still active (no completed_at) is_active = 'completed_at' not in download_info if is_active: logger.debug(f"Download {browser_id} is still active") @@ -590,54 +499,61 @@ def _is_download_active(self, browser_id): return False def _perform_check(self, schedule): - """Perform the actual browser check""" - logger.info(f"Performing scheduled check for {schedule['name']} ({schedule['url']})") - - # Determine duration (20-60s) + """Perform the actual browser check for a schedule. + + Sets status to 'checking' before spawning the thread to prevent + duplicate check threads from being spawned on subsequent loop iterations. + """ + # Guard: mark as checking before the thread starts + schedule['status'] = 'checking' + self._mark_dirty() + duration = random.uniform(20, 60) - - # Thread logic for the check so we don't block main scheduler loop for a minute + check_thread = threading.Thread( target=self._run_browser_check_task, - args=(schedule, duration) + args=(schedule, duration), + daemon=True, ) check_thread.start() - - # Update next check time immediately so we don't spawn multiple + + # Update next check time so the loop doesn't re-trigger immediately self._update_next_check(schedule) def _run_browser_check_task(self, schedule, duration): - """The actual task running in a separate thread""" + """The actual browser check task running in a separate thread.""" browser_id = f"sched_{schedule['id']}_{int(time.time())}" try: - # Queue browser (will wait for previous browsers to close) logger.info(f"Requesting browser for schedule {schedule['id']} (will queue if needed)") - # Use schedule name as filename prefix (without extension) - # This will be combined with timestamp in the filename generator filename_prefix = schedule.get('name') if schedule.get('name') else None success, detector = self.browser_service.start_browser( url=schedule['url'], browser_id=browser_id, - auto_download=True, # Important! - filename=filename_prefix, # Pass name for prefix + auto_download=True, + filename=filename_prefix, resolution=schedule.get('resolution', '1080p'), framerate=schedule.get('framerate', 'any'), output_format=schedule.get('format', 'mp4') ) - + if not success: logger.warning(f"Failed to start browser for schedule {schedule['id']}") + # Revert status so next loop can retry + with self.lock: + for s in self.schedules: + if s['id'] == schedule['id'] and s.get('status') == 'checking': + s['status'] = 'active' + self._mark_dirty() + break return - # Monitor browser for stream detection (up to specified duration) start_wait = time.time() while time.time() - start_wait < duration: status = self.browser_service.get_browser_status(browser_id) if not status: break - # Check if download has started via download service dl_status = self.browser_service.download_service.get_download_status(browser_id) if dl_status: logger.info(f"Download started for schedule {schedule['id']}! (browser_id: {browser_id})") @@ -647,21 +563,35 @@ def _run_browser_check_task(self, schedule, duration): if s['id'] == schedule['id']: s['status'] = 'download_started' s['active_browser_id'] = browser_id + self._mark_dirty() self.save_schedules() break - # Download started successfully, exit monitoring loop break time.sleep(1) - # Close browser after monitoring completes + # If no download started, revert to 'active' so next window check can retry + with self.lock: + for s in self.schedules: + if s['id'] == schedule['id'] and s.get('status') == 'checking': + s['status'] = 'active' + self._mark_dirty() + break + logger.info(f"Closing browser for schedule {schedule['id']}") self.browser_service.close_browser(browser_id) except Exception as e: logger.error(f"Error in browser check task: {e}") + # Revert checking status on error + with self.lock: + for s in self.schedules: + if s['id'] == schedule['id'] and s.get('status') == 'checking': + s['status'] = 'active' + self._mark_dirty() + break try: self.browser_service.close_browser(browser_id) - except: + except Exception: pass diff --git a/app/services/browser_service.py b/app/services/browser_service.py index cb015e2..98bd6a6 100644 --- a/app/services/browser_service.py +++ b/app/services/browser_service.py @@ -152,11 +152,8 @@ def _ensure_chrome_closed(self): except Exception as e: logger.warning(f"Error closing browser {bid}: {e}") - # Wait for Chrome processes to fully terminate - time.sleep(2) - - # Check if Chrome processes are still running - max_wait = 10 # Maximum 10 seconds to wait + # Poll for Chrome to exit — no unconditional sleep upfront + max_wait = 8 # Maximum 8 seconds to wait wait_count = 0 while wait_count < max_wait: @@ -181,8 +178,8 @@ def _ensure_chrome_closed(self): logger.debug(f"Error checking Chrome processes: {e}") break - # Add a few extra seconds buffer to be absolutely sure - time.sleep(3) + # Short buffer to let OS fully release any file locks + time.sleep(1) logger.info("Chrome ready to launch") def _launch_browser_internal(self, url, browser_id, resolution='1080p', framerate='any', auto_download=False, filename=None, output_format='mp4'): diff --git a/app/services/download_service.py b/app/services/download_service.py index 45a00c0..89d9364 100644 --- a/app/services/download_service.py +++ b/app/services/download_service.py @@ -1,4 +1,5 @@ import os +import json import time import logging import subprocess @@ -13,345 +14,374 @@ class DownloadService: def __init__(self, download_dir): self.download_dir = download_dir - self.download_queue = {} - self.direct_download_status = {} - self.download_thumbnails = {} # Cache for thumbnails + self._queue_lock = threading.Lock() + self.download_queue = {} # protected by _queue_lock + self.direct_download_status = {} # protected by _queue_lock + self.download_thumbnails = {} # protected by _queue_lock + self._history_file = os.path.join(download_dir, '.download_history.json') + + # ------------------------------------------------------------------ # + # Public API + # ------------------------------------------------------------------ # def start_download(self, browser_id, stream_url, filename, resolution_name, stream_metadata=None): - """Start a download using FFmpeg""" + """Start a stream download using FFmpeg (called by browser/stream detector).""" output_path = os.path.join(self.download_dir, filename) - - # Start download in background thread threading.Thread( target=self._process_download, args=(browser_id, stream_url, output_path, resolution_name, stream_metadata), daemon=True ).start() - return output_path def start_direct_download(self, browser_id, stream_url, filename): - """Start a direct download with metadata enrichment""" + """Start a direct URL download with metadata enrichment.""" output_path = os.path.join(self.download_dir, filename) - threading.Thread( target=self._direct_download, args=(browser_id, stream_url, output_path), daemon=True ).start() - return browser_id, output_path - def _thumbnail_updater(self, browser_id, stop_event): - """Background thread to update thumbnails periodically""" - logger.debug(f"Starting thumbnail updater for {browser_id}") - - # Check if this is an audio format - skip thumbnails for audio - if browser_id in self.download_queue: - file_path = self.download_queue[browser_id].get('output_path') - if file_path: - ext = os.path.splitext(file_path)[1].lower().lstrip('.') - audio_formats = ['mp3', 'aac', 'm4a', 'flac', 'wav', 'ogg', 'opus', 'wma'] - if ext in audio_formats: - logger.debug(f"Skipping thumbnail generation for audio format: {ext}") - return - - while not stop_event.is_set(): + def stop_download(self, browser_id): + """Stop an active download, escalating from SIGTERM to SIGKILL if needed.""" + with self._queue_lock: + if browser_id not in self.download_queue: + return False + download_info = self.download_queue.pop(browser_id) + self.download_thumbnails.pop(browser_id, None) + + process = download_info.get('process') + if process and process.poll() is None: + logger.debug(f"Terminating download {browser_id}") + process.terminate() try: - if browser_id not in self.download_queue: - break - - download_info = self.download_queue[browser_id] - file_path = download_info.get('output_path') - started_at = download_info.get('started_at', time.time()) + process.wait(timeout=5) + except subprocess.TimeoutExpired: + logger.warning(f"SIGTERM timed out for {browser_id}, escalating to SIGKILL") + process.kill() + process.wait() - # Calculate dynamic seek time (2 seconds behind live edge) - elapsed = max(0, time.time() - started_at) - seek_time = max(0, int(elapsed - 2)) + return True - # Try to extract from file - thumbnail = None - if file_path and os.path.exists(file_path): - # Force cache timeout to 0 if we are seeking to new position to ensure fresh frame, - # effectively bypassing cache for updates, but maybe we want to respect loop interval. - # Since we control the loop, we can just pass cache_timeout=1 - thumbnail = ThumbnailGenerator.extract_thumbnail_from_file( - file_path, - self.download_thumbnails, - browser_id, - cache_timeout=1, - seek_time=seek_time - ) + def get_active_downloads(self): + """Get list of active downloads with progress.""" + with self._queue_lock: + items = list(self.download_queue.items()) - # Update the download info with the new thumbnail - if thumbnail: - download_info['latest_thumbnail'] = thumbnail - # Also update the cache dict so get_active_downloads logic remains consistent if called - self.download_thumbnails[browser_id] = { - 'thumbnail': thumbnail, - 'timestamp': time.time() - } + active = [] + for browser_id, download_info in items: + if 'completed_at' in download_info: + continue - # Smart Wait: - # If we have no thumbnail yet, try eagerly (1s). - # If we have one, update every 10s. - wait_time = 10 if download_info.get('latest_thumbnail') else 1 - - except Exception as e: - logger.error(f"Error in thumbnail updater for {browser_id}: {e}") - wait_time = 10 # Fallback - - # Wait for calculated time or until stopped - if stop_event.wait(wait_time): - break + process = download_info.get('process') + output_path = download_info.get('output_path') + started_at = download_info.get('started_at', time.time()) - logger.debug(f"Stopping thumbnail updater for {browser_id}") + file_size = 0 + if output_path and os.path.exists(output_path): + file_size = os.path.getsize(output_path) - def _process_download(self, browser_id, stream_url, output_path, resolution_name, stream_metadata=None): - """Process download in background thread""" - stop_thumbnail_event = threading.Event() - thumbnail_thread = None - - try: - logger.info(f"Starting FFmpeg download: {stream_url} -> {output_path}") + active.append({ + 'browser_id': browser_id, + 'filename': download_info.get('filename', 'Unknown'), + 'resolution': download_info.get('resolution_name', 'Unknown'), + 'resolution_detail': download_info.get('resolution', 'Unknown'), + 'framerate': download_info.get('framerate', 'Unknown'), + 'codecs': download_info.get('codecs', 'Unknown'), + 'size': file_size, + 'duration': int(time.time() - started_at), + 'is_running': process.poll() is None if process else False, + 'thumbnail': download_info.get('latest_thumbnail'), + }) - # Get metadata for display - if stream_metadata: - resolution_display = stream_metadata.get('resolution') or stream_metadata.get('name', 'Unknown') - if stream_metadata.get('resolution') and 'x' in str(stream_metadata.get('resolution')): - fps = stream_metadata.get('framerate', '').split('.')[0] if stream_metadata.get('framerate') else '' - resolution_display = f"{stream_metadata.get('resolution')}@{fps}fps" if fps else stream_metadata.get('resolution') - elif stream_metadata.get('name'): - resolution_display = stream_metadata.get('name') - metadata = stream_metadata - else: - resolution_display = 'Unknown' - metadata = {} + return active - # Start FFmpeg process - process = self._start_ffmpeg_process(stream_url, output_path) + def get_download_status(self, browser_id): + """Get download status for a specific browser_id.""" + with self._queue_lock: + download_info = self.download_queue.get(browser_id) - # Store process info - self.download_queue[browser_id] = { - 'process': process, - 'output_path': output_path, - 'stream_url': stream_url, - 'started_at': time.time(), - 'resolution_name': resolution_display, - 'resolution': metadata.get('resolution', 'Unknown'), - 'framerate': metadata.get('framerate', 'Unknown'), - 'codecs': metadata.get('codecs', 'Unknown'), - 'filename': os.path.basename(output_path), - 'latest_thumbnail': None - } - - # Start thumbnail updater - thumbnail_thread = threading.Thread( - target=self._thumbnail_updater, - args=(browser_id, stop_thumbnail_event), - daemon=True - ) - thumbnail_thread.start() + if download_info is None: + return None - # Wait for completion - stdout, stderr = process.communicate() + if 'completed_at' in download_info: + duration = download_info['completed_at'] - download_info['started_at'] + else: + duration = time.time() - download_info['started_at'] + + return { + 'output_path': download_info['output_path'], + 'stream_url': download_info['stream_url'], + 'duration': duration, + 'completed': 'completed_at' in download_info, + 'success': download_info.get('success', True), + } + + def get_history(self): + """Return the persisted download history list.""" + try: + if os.path.exists(self._history_file): + with open(self._history_file, 'r') as f: + return json.load(f) + except Exception as e: + logger.error(f"Error reading download history: {e}") + return [] - # Mark as completed - if browser_id in self.download_queue: - self.download_queue[browser_id]['completed_at'] = time.time() - self.download_queue[browser_id]['success'] = (process.returncode == 0) + # ------------------------------------------------------------------ # + # Internal helpers + # ------------------------------------------------------------------ # - if process.returncode == 0: - logger.info(f"Download completed: {output_path}") - else: - logger.error(f"FFmpeg error: {stderr}") + def _process_download(self, browser_id, stream_url, output_path, resolution_name, stream_metadata=None): + """Thin wrapper: compute display name then delegate to shared core.""" + if stream_metadata: + resolution_display = stream_metadata.get('resolution') or stream_metadata.get('name', 'Unknown') + if stream_metadata.get('resolution') and 'x' in str(stream_metadata.get('resolution')): + fps = stream_metadata.get('framerate', '').split('.')[0] if stream_metadata.get('framerate') else '' + resolution_display = ( + f"{stream_metadata['resolution']}@{fps}fps" if fps else stream_metadata['resolution'] + ) + elif stream_metadata.get('name'): + resolution_display = stream_metadata['name'] + else: + resolution_display = resolution_name or 'Unknown' + stream_metadata = {} - except Exception as e: - logger.error(f"Download failed: {e}") - if browser_id in self.download_queue: - self.download_queue[browser_id]['completed_at'] = time.time() - self.download_queue[browser_id]['success'] = False - finally: - # Stop thumbnail updater - if stop_thumbnail_event: - stop_thumbnail_event.set() - - # Clean up completed download from queue after a short delay - # This allows get_download_status() to work for scheduler, then removes it - def cleanup_after_delay(): - time.sleep(30) # Wait 30 seconds - if browser_id in self.download_queue and 'completed_at' in self.download_queue[browser_id]: - logger.debug(f"Cleaning up completed download from queue: {browser_id}") - del self.download_queue[browser_id] - # Also clean up thumbnail cache - if browser_id in self.download_thumbnails: - del self.download_thumbnails[browser_id] - - threading.Thread(target=cleanup_after_delay, daemon=True).start() + self._run_download_core(browser_id, stream_url, output_path, resolution_display, stream_metadata) def _direct_download(self, browser_id, stream_url, output_path): - """Execute direct download with metadata enrichment""" - stop_thumbnail_event = threading.Event() - + """Enrich metadata, generate thumbnail, then run shared core download.""" try: logger.info(f"Starting direct download: {stream_url[:100]}...") - # Create stream entry for metadata enrichment stream_entry = { 'url': stream_url, 'bandwidth': 0, 'resolution': '', 'framerate': '', 'codecs': '', - 'name': 'direct' + 'name': 'direct', } - - # Enrich metadata - logger.debug("Enriching stream metadata for direct download...") MetadataExtractor.enrich_stream_metadata(stream_entry) - # Generate thumbnail - logger.debug("Generating thumbnail for direct download...") thumbnail = ThumbnailGenerator.generate_stream_thumbnail(stream_url) - - # Clean thumbnail for storage thumbnail_data = None if thumbnail and thumbnail.startswith('data:image/'): - thumbnail_data = thumbnail.split(',', 1)[1] + thumbnail_data = thumbnail.split(',', 1)[1] elif thumbnail: - thumbnail_data = thumbnail + thumbnail_data = thumbnail - # Prepare metadata resolution_display = stream_entry.get('resolution', 'Unknown') if stream_entry.get('resolution') and 'x' in str(stream_entry.get('resolution')): fps = stream_entry.get('framerate', '').split('.')[0] if stream_entry.get('framerate') else '' - resolution_display = f"{stream_entry.get('resolution')}@{fps}fps" if fps else stream_entry.get('resolution') + resolution_display = ( + f"{stream_entry['resolution']}@{fps}fps" if fps else stream_entry['resolution'] + ) + + with self._queue_lock: + self.direct_download_status[browser_id] = { + 'browser_id': browser_id, + 'is_running': True, + 'download_started': True, + 'thumbnail': thumbnail_data, + 'selected_stream_metadata': stream_entry, + } + + self._run_download_core( + browser_id, stream_url, output_path, + resolution_display, stream_entry, + initial_thumbnail=thumbnail_data, + ) - # Store status - self.direct_download_status[browser_id] = { - 'browser_id': browser_id, - 'is_running': True, - 'download_started': True, - 'thumbnail': thumbnail_data, - 'selected_stream_metadata': stream_entry - } + except Exception as e: + logger.error(f"Direct download error: {e}") + with self._queue_lock: + if browser_id in self.download_queue: + self.download_queue[browser_id]['completed_at'] = time.time() + self.download_queue[browser_id]['success'] = False + finally: + with self._queue_lock: + self.direct_download_status.pop(browser_id, None) - # Start FFmpeg + def _run_download_core(self, browser_id, stream_url, output_path, resolution_display, metadata, initial_thumbnail=None): + """Core download execution: FFmpeg subprocess, thumbnail updates, history write.""" + stop_thumbnail_event = threading.Event() + started_at = time.time() + + try: + logger.info(f"Starting FFmpeg download: {stream_url} -> {output_path}") process = self._start_ffmpeg_process(stream_url, output_path) - # Store in queue - self.download_queue[browser_id] = { + queue_entry = { 'process': process, 'output_path': output_path, 'stream_url': stream_url, - 'started_at': time.time(), + 'started_at': started_at, 'resolution_name': resolution_display, - 'resolution': stream_entry.get('resolution', 'Unknown'), - 'framerate': stream_entry.get('framerate', 'Unknown'), - 'codecs': stream_entry.get('codecs', 'Unknown'), + 'resolution': metadata.get('resolution', 'Unknown'), + 'framerate': metadata.get('framerate', 'Unknown'), + 'codecs': metadata.get('codecs', 'Unknown'), 'filename': os.path.basename(output_path), - 'latest_thumbnail': thumbnail_data + 'latest_thumbnail': initial_thumbnail, } - - # Start thumbnail updater - threading.Thread( + with self._queue_lock: + self.download_queue[browser_id] = queue_entry + + thumbnail_thread = threading.Thread( target=self._thumbnail_updater, args=(browser_id, stop_thumbnail_event), - daemon=True - ).start() + daemon=True, + ) + thumbnail_thread.start() stdout, stderr = process.communicate() - # Mark as completed - if browser_id in self.download_queue: - self.download_queue[browser_id]['completed_at'] = time.time() - self.download_queue[browser_id]['success'] = (process.returncode == 0) + success = process.returncode == 0 + completed_at = time.time() + + with self._queue_lock: + if browser_id in self.download_queue: + self.download_queue[browser_id]['completed_at'] = completed_at + self.download_queue[browser_id]['success'] = success - if process.returncode == 0: - logger.info(f"Direct download completed: {output_path}") + if success: + logger.info(f"Download completed: {output_path}") else: - logger.error(f"Direct download failed: {stderr}") + logger.error(f"FFmpeg error (rc={process.returncode}): {stderr}") - # Clean up status - if browser_id in self.direct_download_status: - del self.direct_download_status[browser_id] + self._append_history({ + 'browser_id': browser_id, + 'filename': os.path.basename(output_path), + 'stream_url': stream_url, + 'resolution': metadata.get('resolution', 'Unknown'), + 'framerate': metadata.get('framerate', 'Unknown'), + 'started_at': started_at, + 'completed_at': completed_at, + 'duration_s': round(completed_at - started_at, 1), + 'success': success, + 'file_size': os.path.getsize(output_path) if os.path.exists(output_path) else 0, + }) except Exception as e: - logger.error(f"Direct download error: {e}") - if browser_id in self.download_queue: - self.download_queue[browser_id]['completed_at'] = time.time() - self.download_queue[browser_id]['success'] = False + logger.error(f"Download failed: {e}") + with self._queue_lock: + if browser_id in self.download_queue: + self.download_queue[browser_id]['completed_at'] = time.time() + self.download_queue[browser_id]['success'] = False finally: stop_thumbnail_event.set() + # Use a Timer instead of spawning a thread just to sleep + threading.Timer(30.0, self._cleanup_download, args=(browser_id,)).start() + + def _cleanup_download(self, browser_id): + """Remove a completed download entry from the queue (called by Timer after 30 s).""" + with self._queue_lock: + info = self.download_queue.get(browser_id) + if info and 'completed_at' in info: + logger.debug(f"Cleaning up completed download from queue: {browser_id}") + del self.download_queue[browser_id] + self.download_thumbnails.pop(browser_id, None) - # Clean up completed download from queue after a short delay - # This allows get_download_status() to work for scheduler, then removes it - def cleanup_after_delay(): - time.sleep(30) # Wait 30 seconds - if browser_id in self.download_queue and 'completed_at' in self.download_queue[browser_id]: - logger.debug(f"Cleaning up completed download from queue: {browser_id}") - del self.download_queue[browser_id] - # Also clean up thumbnail cache - if browser_id in self.download_thumbnails: - del self.download_thumbnails[browser_id] + def _thumbnail_updater(self, browser_id, stop_event): + """Background thread to periodically extract and cache a live thumbnail.""" + logger.debug(f"Starting thumbnail updater for {browser_id}") + + # Skip audio-only formats up front + with self._queue_lock: + file_path = self.download_queue.get(browser_id, {}).get('output_path', '') + if file_path: + ext = os.path.splitext(file_path)[1].lower().lstrip('.') + if ext in {'mp3', 'aac', 'm4a', 'flac', 'wav', 'ogg', 'opus', 'wma'}: + logger.debug(f"Skipping thumbnail generation for audio format: {ext}") + return + + while not stop_event.is_set(): + try: + with self._queue_lock: + if browser_id not in self.download_queue: + break + info = self.download_queue[browser_id] + file_path = info.get('output_path') + started_at = info.get('started_at', time.time()) + has_thumbnail = bool(info.get('latest_thumbnail')) - threading.Thread(target=cleanup_after_delay, daemon=True).start() + elapsed = max(0, time.time() - started_at) + seek_time = max(0, int(elapsed - 2)) + + thumbnail = None + if file_path and os.path.exists(file_path): + thumbnail = ThumbnailGenerator.extract_thumbnail_from_file( + file_path, + self.download_thumbnails, + browser_id, + cache_timeout=1, + seek_time=seek_time, + ) + + if thumbnail: + with self._queue_lock: + if browser_id in self.download_queue: + self.download_queue[browser_id]['latest_thumbnail'] = thumbnail + self.download_thumbnails[browser_id] = { + 'thumbnail': thumbnail, + 'timestamp': time.time(), + } + has_thumbnail = True + + wait_time = 10 if has_thumbnail else 1 + + except Exception as e: + logger.error(f"Error in thumbnail updater for {browser_id}: {e}") + wait_time = 10 + + if stop_event.wait(wait_time): + break + + logger.debug(f"Stopping thumbnail updater for {browser_id}") + + def _append_history(self, entry): + """Append a completed download record to the persistent history file.""" + try: + history = [] + if os.path.exists(self._history_file): + try: + with open(self._history_file, 'r') as f: + history = json.load(f) + except Exception: + history = [] + history.append(entry) + with open(self._history_file, 'w') as f: + json.dump(history, f, indent=2) + except Exception as e: + logger.error(f"Error writing download history: {e}") def _start_ffmpeg_process(self, stream_url, output_path): - """Start FFmpeg process for downloading""" - # Determine output format from extension + """Build and launch the appropriate FFmpeg command for the given output format.""" ext = os.path.splitext(output_path)[1].lower().lstrip('.') - - # Audio-only formats - audio_formats = ['mp3', 'aac', 'm4a', 'flac', 'wav', 'ogg', 'opus', 'wma'] + audio_formats = {'mp3', 'aac', 'm4a', 'flac', 'wav', 'ogg', 'opus', 'wma'} if ext in audio_formats: - # Audio extraction - need to encode - cmd = [ - 'ffmpeg', - '-loglevel', 'error', # Only show errors - '-i', stream_url, - '-vn', # No video - ] - - # Format-specific encoding options - if ext == 'mp3': - cmd.extend(['-c:a', 'libmp3lame', '-q:a', '2']) - elif ext == 'aac': - cmd.extend(['-c:a', 'aac', '-b:a', '192k']) - elif ext == 'm4a': - cmd.extend(['-c:a', 'aac', '-b:a', '192k']) - elif ext == 'flac': - cmd.extend(['-c:a', 'flac']) - elif ext == 'wav': - cmd.extend(['-c:a', 'pcm_s16le']) - elif ext == 'ogg': - cmd.extend(['-c:a', 'libvorbis', '-q:a', '6']) - elif ext == 'opus': - cmd.extend(['-c:a', 'libopus', '-b:a', '128k']) - elif ext == 'wma': - cmd.extend(['-c:a', 'wmav2', '-b:a', '192k']) - - cmd.extend(['-y', output_path]) + cmd = ['ffmpeg', '-loglevel', 'error', '-i', stream_url, '-vn'] + codec_map = { + 'mp3': ['-c:a', 'libmp3lame', '-q:a', '2'], + 'aac': ['-c:a', 'aac', '-b:a', '192k'], + 'm4a': ['-c:a', 'aac', '-b:a', '192k'], + 'flac': ['-c:a', 'flac'], + 'wav': ['-c:a', 'pcm_s16le'], + 'ogg': ['-c:a', 'libvorbis', '-q:a', '6'], + 'opus': ['-c:a', 'libopus', '-b:a', '128k'], + 'wma': ['-c:a', 'wmav2', '-b:a', '192k'], + } + cmd.extend(codec_map.get(ext, ['-c:a', 'copy'])) else: - # Video formats - try stream copy first - cmd = [ - 'ffmpeg', - '-loglevel', 'error', # Only show errors - '-i', stream_url, - ] - - # Format-specific options - if ext in ['mp4', 'm4v', 'mov']: - cmd.extend([ - '-c', 'copy', - '-bsf:a', 'aac_adtstoasc', - '-movflags', '+frag_keyframe+empty_moov', - ]) + cmd = ['ffmpeg', '-loglevel', 'error', '-i', stream_url] + if ext in ('mp4', 'm4v', 'mov'): + cmd.extend(['-c', 'copy', '-bsf:a', 'aac_adtstoasc', + '-movflags', '+frag_keyframe+empty_moov']) elif ext == 'mkv': cmd.extend(['-c', 'copy']) elif ext == 'webm': - # WebM may need re-encoding if source isn't VP8/VP9 cmd.extend(['-c:v', 'copy', '-c:a', 'copy']) elif ext == 'ts': cmd.extend(['-c', 'copy', '-bsf:v', 'h264_mp4toannexb']) @@ -362,91 +392,13 @@ def _start_ffmpeg_process(self, stream_url, output_path): elif ext == 'avi': cmd.extend(['-c', 'copy']) else: - # Default: stream copy cmd.extend(['-c', 'copy']) - - cmd.extend(['-y', output_path]) + + cmd.extend(['-y', output_path]) return subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True + universal_newlines=True, ) - - def get_active_downloads(self): - """Get list of active downloads with progress""" - active = [] - - for browser_id, download_info in list(self.download_queue.items()): - # Skip completed downloads - if 'completed_at' in download_info: - continue - - process = download_info.get('process') - output_path = download_info.get('output_path') - started_at = download_info.get('started_at') - - # Check file size - file_size = 0 - if os.path.exists(output_path): - file_size = os.path.getsize(output_path) - - # Calculate duration - duration = int(time.time() - started_at) - - # Check if process is still running - is_running = process.poll() is None if process else False - - # Use cached thumbnail managed by background thread - thumbnail = download_info.get('latest_thumbnail') - - active.append({ - 'browser_id': browser_id, - 'filename': download_info.get('filename', 'Unknown'), - 'resolution': download_info.get('resolution_name', 'Unknown'), - 'resolution_detail': download_info.get('resolution', 'Unknown'), - 'framerate': download_info.get('framerate', 'Unknown'), - 'codecs': download_info.get('codecs', 'Unknown'), - 'size': file_size, - 'duration': duration, - 'is_running': is_running, - 'thumbnail': thumbnail - }) - - return active - - def stop_download(self, browser_id): - """Stop an active download""" - if browser_id in self.download_queue: - download_info = self.download_queue[browser_id] - process = download_info.get('process') - - if process and process.poll() is None: - logger.debug(f"Stopping download for browser {browser_id}") - process.terminate() - process.wait(timeout=5) - logger.debug(f"Download stopped for browser {browser_id}") - - del self.download_queue[browser_id] - return True - return False - - def get_download_status(self, browser_id): - """Get download status for a specific browser_id""" - if browser_id in self.download_queue: - download_info = self.download_queue[browser_id] - # Calculate duration - if 'completed_at' in download_info: - duration = download_info['completed_at'] - download_info['started_at'] - else: - duration = time.time() - download_info['started_at'] - - return { - 'output_path': download_info['output_path'], - 'stream_url': download_info['stream_url'], - 'duration': duration, - 'completed': 'completed_at' in download_info, - 'success': download_info.get('success', True) - } - return None diff --git a/app/static/js/downloads-browser.js b/app/static/js/downloads-browser.js index 36956f0..c464ae1 100644 --- a/app/static/js/downloads-browser.js +++ b/app/static/js/downloads-browser.js @@ -250,86 +250,107 @@ async function pollDirectDownloadStatus(browserId) { checkStatus(); } -// Poll browser status -function startStatusPolling() { - if (AppState.statusCheckInterval) { - clearInterval(AppState.statusCheckInterval); +// Handle a browser status data payload (shared by SSE and polling fallback) +function handleBrowserStatus(data) { + // Show stream selection modal if streams are available and manual mode + if (data.awaiting_resolution_selection && data.available_resolutions && data.available_resolutions.length > 0) { + console.log('=== AVAILABLE STREAMS ===', data.available_resolutions.length); + + const vncContainer = document.getElementById('vnc-container'); + if (!vncContainer.classList.contains('active')) { + const vncFrame = document.getElementById('vnc-frame'); + const vncUrl = 'http://' + window.location.hostname + ':6080/vnc.html?autoconnect=true&resize=scale'; + vncFrame.src = vncUrl; + vncContainer.classList.add('active'); + setTimeout(() => { + vncContainer.scrollIntoView({ behavior: 'smooth', block: 'start' }); + }, 500); + } + + showStreamModal(data.available_resolutions); + + const statusBox = document.getElementById('browser-status'); + showStatus(statusBox, `✓ Found ${data.available_resolutions.length} streams - Select one to download`, 'success'); } - AppState.statusCheckInterval = setInterval(async () => { - if (!AppState.currentBrowserId) return; + // Handle download started (both auto and manual) + if (data.download_started && !AppState.downloadPopupShown) { + AppState.downloadPopupShown = true; + const statusBox = document.getElementById('browser-status'); + showStatus(statusBox, '✓ Download started!', 'success'); + showDownloadStartedPopup(data.selected_stream_metadata, data.thumbnail); + setTimeout(() => loadDownloads(), 2000); + } - try { - const response = await fetch(`/api/browser/status/${AppState.currentBrowserId}`); - const data = await response.json(); + if (!data.is_running) { + console.log('Browser stopped'); + stopStatusPolling(); - // DEBUG: Log status data - console.log('=== BROWSER STATUS UPDATE ==='); - console.log('Browser ID:', AppState.currentBrowserId); - console.log('Is Running:', data.is_running); - console.log('Download Started:', data.download_started); - console.log('Detected Streams Count:', data.detected_streams); - console.log('Awaiting Resolution Selection:', data.awaiting_resolution_selection); - - // Show stream selection modal if streams are available and manual mode - if (data.awaiting_resolution_selection && data.available_resolutions && data.available_resolutions.length > 0) { - console.log('=== AVAILABLE STREAMS ==='); - console.log('Count:', data.available_resolutions.length); - - data.available_resolutions.forEach((res, index) => { - console.log(`--- Stream ${index + 1} ---`); - console.log('Name:', res.name); - console.log('Resolution:', res.resolution); - console.log('Framerate:', res.framerate); - }); - - // Show VNC browser if not already shown (needed for manual selection) - const vncContainer = document.getElementById('vnc-container'); - if (!vncContainer.classList.contains('active')) { - const vncFrame = document.getElementById('vnc-frame'); - const vncUrl = 'http://' + window.location.hostname + ':6080/vnc.html?autoconnect=true&resize=scale'; - vncFrame.src = vncUrl; - vncContainer.classList.add('active'); - setTimeout(() => { - vncContainer.scrollIntoView({ behavior: 'smooth', block: 'start' }); - }, 500); - } - - // Show modal with all available streams - showStreamModal(data.available_resolutions); - - const statusBox = document.getElementById('browser-status'); - showStatus(statusBox, `✓ Found ${data.available_resolutions.length} streams - Select one to download`, 'success'); - } + const btn = document.getElementById('browser-start-btn'); + btn.disabled = false; + btn.innerHTML = 'Open Browser & Detect'; - // Handle download started (both auto and manual) - if (data.download_started && !AppState.downloadPopupShown) { - console.log('✓ Download started'); - AppState.downloadPopupShown = true; - const statusBox = document.getElementById('browser-status'); - showStatus(statusBox, '✓ Download started!', 'success'); + const vncContainer = document.getElementById('vnc-container'); + vncContainer.classList.remove('active'); - // Show download confirmation popup - showDownloadStartedPopup(data.selected_stream_metadata, data.thumbnail); + AppState.currentBrowserId = null; + loadDownloads(); + } +} - setTimeout(() => loadDownloads(), 2000); - } +// Stop any active status polling or SSE connection +function stopStatusPolling() { + if (AppState.statusCheckInterval) { + clearInterval(AppState.statusCheckInterval); + AppState.statusCheckInterval = null; + } + if (AppState.statusEventSource) { + AppState.statusEventSource.close(); + AppState.statusEventSource = null; + } +} - if (!data.is_running) { - console.log('!!! BROWSER STOPPED !!!'); - clearInterval(AppState.statusCheckInterval); +// Start browser status updates via SSE, with polling fallback +function startStatusPolling() { + stopStatusPolling(); - // Reset button and hide VNC - const btn = document.getElementById('browser-start-btn'); - btn.disabled = false; - btn.innerHTML = 'Open Browser & Detect'; + const browserId = AppState.currentBrowserId; - const vncContainer = document.getElementById('vnc-container'); - vncContainer.classList.remove('active'); + if (typeof EventSource !== 'undefined') { + const evtSource = new EventSource(`/api/events/browser/${browserId}`); + AppState.statusEventSource = evtSource; - AppState.currentBrowserId = null; - loadDownloads(); + evtSource.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + console.log('=== BROWSER STATUS UPDATE (SSE) ===', data.is_running, data.download_started); + handleBrowserStatus(data); + } catch (e) { + console.error('SSE parse error:', e); } + }; + + evtSource.onerror = () => { + console.warn('SSE connection failed, falling back to polling'); + evtSource.close(); + AppState.statusEventSource = null; + _startPollingFallback(browserId); + }; + } else { + _startPollingFallback(browserId); + } +} + +// Polling fallback when SSE is unavailable or fails +function _startPollingFallback(browserId) { + AppState.statusCheckInterval = setInterval(async () => { + if (!AppState.currentBrowserId) return; + + try { + const response = await fetch(`/api/browser/status/${browserId}`); + const data = await response.json(); + console.log('=== BROWSER STATUS UPDATE (poll) ===', data.is_running, data.download_started); + handleBrowserStatus(data); } catch (error) { console.error('Status check error:', error); } diff --git a/app/static/js/state.js b/app/static/js/state.js index 59a1b3e..1a18fbf 100644 --- a/app/static/js/state.js +++ b/app/static/js/state.js @@ -2,6 +2,7 @@ const AppState = { currentBrowserId: null, statusCheckInterval: null, + statusEventSource: null, // SSE EventSource for browser status updates countdownInterval: null, countdownValue: 15, debugLogContent: '', From 92001c2919def6944960c98663f1444037382778 Mon Sep 17 00:00:00 2001 From: DorkSoul <71715506+DorkSoul@users.noreply.github.com> Date: Tue, 17 Mar 2026 14:01:56 +0000 Subject: [PATCH 2/7] docker node --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2e65f96..c3884d9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -10,6 +10,7 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: build-and-push: From e129b815c17c608499234d78994b5a6c9795cd00 Mon Sep 17 00:00:00 2001 From: DorkSoul <71715506+DorkSoul@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:18:47 +0000 Subject: [PATCH 3/7] svg --- app/app.py | 2 +- app/config.py | 5 +++ app/services/download_service.py | 6 ++- app/static/icons/mars.svg | 68 ++++++++++++++++++++++++++++++++ app/templates/index.html | 10 +++-- 5 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 app/static/icons/mars.svg diff --git a/app/app.py b/app/app.py index 8ca2c55..a3d3044 100644 --- a/app/app.py +++ b/app/app.py @@ -19,7 +19,7 @@ def create_app(): config.log_startup_info(logger) # Initialize services - download_service = DownloadService(config.DOWNLOAD_DIR) + download_service = DownloadService(config.DOWNLOAD_DIR, history_file=config.HISTORY_FILE) browser_service = BrowserService(config, download_service) # Initialize Scheduler diff --git a/app/config.py b/app/config.py index 7cc08ca..97b4578 100644 --- a/app/config.py +++ b/app/config.py @@ -16,6 +16,10 @@ def __init__(self): # Schedules self.SCHEDULES_FILE = os.path.join(self.CHROME_USER_DATA_DIR, 'schedules.json') + # Logs / history + self.LOGS_DIR = '/app/logs' + self.HISTORY_FILE = os.path.join(self.LOGS_DIR, 'download_history.json') + # Chrome paths self.CHROMEDRIVER_PATH = '/usr/local/bin/chromedriver' self.CHROMEDRIVER_LOG_PATH = '/app/logs/chromedriver.log' @@ -48,6 +52,7 @@ def check_directories(self): """Ensure required directories exist""" os.makedirs(self.DOWNLOAD_DIR, exist_ok=True) os.makedirs(self.CHROME_USER_DATA_DIR, exist_ok=True) + os.makedirs(self.LOGS_DIR, exist_ok=True) def log_startup_info(self, logger): """Log startup information""" diff --git a/app/services/download_service.py b/app/services/download_service.py index 89d9364..e14243e 100644 --- a/app/services/download_service.py +++ b/app/services/download_service.py @@ -12,13 +12,15 @@ class DownloadService: """Manages video downloads using FFmpeg""" - def __init__(self, download_dir): + def __init__(self, download_dir, history_file=None): self.download_dir = download_dir self._queue_lock = threading.Lock() self.download_queue = {} # protected by _queue_lock self.direct_download_status = {} # protected by _queue_lock self.download_thumbnails = {} # protected by _queue_lock - self._history_file = os.path.join(download_dir, '.download_history.json') + # History file lives in /app/logs by default so it doesn't appear + # inside the user's downloads folder. Can be overridden via history_file. + self._history_file = history_file or os.path.join(download_dir, '.download_history.json') # ------------------------------------------------------------------ # # Public API diff --git a/app/static/icons/mars.svg b/app/static/icons/mars.svg new file mode 100644 index 0000000..3d45844 --- /dev/null +++ b/app/static/icons/mars.svg @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/app/templates/index.html b/app/templates/index.html index 3cee347..7cb7776 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -5,13 +5,15 @@ MARS - Media Archive Recording System + +
-

🎬 MARS

+

Mars MARS

Media Archive Recording System - Intelligent stream recording and archiving

@@ -272,12 +274,12 @@

Add New Sche

-
-
+
+

📋 Active Schedules

-
+
From 1ff2bb5b0e78021e31851031ff651477b5bbe01c Mon Sep 17 00:00:00 2001 From: DorkSoul <71715506+DorkSoul@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:25:12 +0000 Subject: [PATCH 4/7] sizing --- app/templates/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/templates/index.html b/app/templates/index.html index 7cb7776..5b7754f 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -274,12 +274,12 @@

Add New Sche

-
+

📋 Active Schedules

-
+
From bd7338248ffb4583a22316954ebe58ca60f4bccd Mon Sep 17 00:00:00 2001 From: DorkSoul <71715506+DorkSoul@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:47:11 +0000 Subject: [PATCH 5/7] sizing --- app/static/css/style.css | 41 ++++++++++++++++++++++++++++++++++++++++ app/templates/index.html | 12 ++++++------ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/app/static/css/style.css b/app/static/css/style.css index 1e6a15a..8258849 100644 --- a/app/static/css/style.css +++ b/app/static/css/style.css @@ -46,6 +46,47 @@ header p { } } +/* Schedule row — spans both grid columns so the pair forms its own flex row. + The form card determines the height; the active-schedules card stretches + to match without letting its list content expand the row. */ +.schedule-row { + grid-column: 1 / -1; + display: flex; + gap: 20px; + align-items: stretch; +} + +.schedule-form-card { + flex: none; + width: calc(50% - 10px); +} + +.schedule-active-card { + flex: 1; + min-height: 0; /* don't let content drive the flex-line height */ + overflow: hidden; /* clip so content can't push the card taller */ + display: flex; + flex-direction: column; +} + +.schedule-active-card #schedules-list { + flex: 1; + min-height: 0; + overflow-y: auto; + display: grid; + align-content: start; + gap: 10px; +} + +@media (max-width: 968px) { + .schedule-row { + flex-direction: column; + } + .schedule-form-card { + width: 100%; + } +} + .card { background: #2d2d44; border-radius: 12px; diff --git a/app/templates/index.html b/app/templates/index.html index 5b7754f..17b6e99 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -171,10 +171,9 @@

🖥️ Browser View

- - - -
+ +
+

📅 Scheduled Downloads

Schedule automatic stream checks. The browser will open the page and search for video streams.

@@ -274,15 +273,16 @@

Add New Sche

-
+

📋 Active Schedules

-
+
+
From 3224893dea28b322b0a6ec3c78de137c5a274132 Mon Sep 17 00:00:00 2001 From: DorkSoul <71715506+DorkSoul@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:59:08 +0000 Subject: [PATCH 6/7] sizing --- app/static/css/style.css | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/app/static/css/style.css b/app/static/css/style.css index 8258849..ed3bea6 100644 --- a/app/static/css/style.css +++ b/app/static/css/style.css @@ -46,14 +46,17 @@ header p { } } -/* Schedule row — spans both grid columns so the pair forms its own flex row. - The form card determines the height; the active-schedules card stretches - to match without letting its list content expand the row. */ +/* Schedule row — spans both grid columns. + The form card (left) sizes to its own content and drives the row height. + The active-schedules card (right) is position:absolute so it never + contributes to the row height — it just pins to top/bottom of the wrapper, + matching the form card exactly, and scrolls internally when full. */ .schedule-row { grid-column: 1 / -1; + position: relative; /* establishes containing block for the abs-pos card */ display: flex; + align-items: flex-start; gap: 20px; - align-items: stretch; } .schedule-form-card { @@ -62,11 +65,14 @@ header p { } .schedule-active-card { - flex: 1; - min-height: 0; /* don't let content drive the flex-line height */ - overflow: hidden; /* clip so content can't push the card taller */ + position: absolute; + top: 0; + bottom: 0; + right: 0; + width: calc(50% - 10px); display: flex; flex-direction: column; + overflow: hidden; } .schedule-active-card #schedules-list { @@ -85,6 +91,11 @@ header p { .schedule-form-card { width: 100%; } + .schedule-active-card { + position: static; + width: 100%; + max-height: 50vh; + } } .card { From e4c77a3b093ff9869d924fc275564c1ec8311e3f Mon Sep 17 00:00:00 2001 From: DorkSoul <71715506+DorkSoul@users.noreply.github.com> Date: Tue, 17 Mar 2026 16:08:15 +0000 Subject: [PATCH 7/7] pause button --- app/routes/scheduler_routes.py | 17 +++++++++++++++++ app/scheduler.py | 28 +++++++++++++++++++++++++++- app/static/js/schedules.js | 30 +++++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/app/routes/scheduler_routes.py b/app/routes/scheduler_routes.py index 43a78de..1bc83b2 100644 --- a/app/routes/scheduler_routes.py +++ b/app/routes/scheduler_routes.py @@ -96,6 +96,23 @@ def update_schedule(schedule_id): logger.error(f"Error updating schedule: {e}") return jsonify({'error': str(e)}), 500 + @scheduler_bp.route('//pause', methods=['POST']) + def pause_schedule(schedule_id): + """Toggle the paused state of a schedule""" + try: + updated = scheduler.pause_schedule(schedule_id) + if updated: + return jsonify({ + 'success': True, + 'schedule': updated, + 'paused': updated.get('paused', False) + }) + else: + return jsonify({'error': 'Schedule not found'}), 404 + except Exception as e: + logger.error(f"Error toggling pause for schedule {schedule_id}: {e}") + return jsonify({'error': str(e)}), 500 + @scheduler_bp.route('/refresh', methods=['POST']) def refresh_schedules(): """Force refresh all schedule next_check times""" diff --git a/app/scheduler.py b/app/scheduler.py index 12e414e..e166b26 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -171,11 +171,31 @@ def move_to_next_slot(self, browser_id): return False + def pause_schedule(self, schedule_id): + """Toggle the paused state of a schedule.""" + with self.lock: + for schedule in self.schedules: + if schedule['id'] == schedule_id: + currently_paused = schedule.get('paused', False) + schedule['paused'] = not currently_paused + if schedule['paused']: + schedule['status'] = 'paused' + else: + # Resuming — recalculate next check and set pending + schedule['status'] = 'pending' + self._update_next_check(schedule) + self._mark_dirty() + self.save_schedules() + logger.info(f"Schedule {schedule_id} {'paused' if schedule['paused'] else 'unpaused'}") + return schedule + return None + def get_schedules(self): - """Get all schedules, sorted by next_check time""" + """Get all schedules — active schedules sorted by next_check, paused schedules at the bottom.""" sorted_schedules = sorted( self.schedules, key=lambda s: ( + s.get('paused', False), # paused go last s.get('next_check') is None, s.get('next_check') or '' ) @@ -192,6 +212,8 @@ def refresh_all_schedule_times(self): with self.lock: count = 0 for schedule in self.schedules: + if schedule.get('paused', False): + continue # don't disturb paused schedules schedule['status'] = 'pending' schedule['active_browser_id'] = None schedule['manual_stop'] = False @@ -241,6 +263,10 @@ def _check_schedules(self): with self.lock: for schedule in self.schedules: + # Skip paused schedules entirely + if schedule.get('paused', False): + continue + if schedule['status'] == 'completed' and not schedule.get('daily'): continue diff --git a/app/static/js/schedules.js b/app/static/js/schedules.js index f87cb46..96bd16b 100644 --- a/app/static/js/schedules.js +++ b/app/static/js/schedules.js @@ -13,9 +13,11 @@ async function loadSchedules() { return; } - // Sort schedules by next_check time (soonest first) - // For schedules without next_check, use start_time + // Sort schedules: active first by next_check, paused always at bottom schedules.sort((a, b) => { + if (a.paused && !b.paused) return 1; + if (!a.paused && b.paused) return -1; + // Helper function to get sortable timestamp const getSortTime = (sched) => { // If next_check exists, use it (it's already calculated by backend) @@ -86,7 +88,10 @@ async function loadSchedules() { let statusColor = '#666'; let statusText = sched.status; - if (sched.status === 'active') { + if (sched.paused) { + statusColor = '#888'; + statusText = 'paused'; + } else if (sched.status === 'active') { statusColor = '#28a745'; const nextCheck = sched.next_check ? formatRFC2822(sched.next_check) : 'Pending window'; statusText = `${sched.status} (Next check: ${nextCheck})`; @@ -115,6 +120,7 @@ async function loadSchedules() {
+
@@ -242,6 +248,24 @@ async function addSchedule() { } } +async function pauseSchedule(id, btn) { + try { + btn.disabled = true; + const response = await fetch(`/api/schedules/${id}/pause`, { method: 'POST' }); + const data = await response.json(); + + if (data.success) { + loadSchedules(); + } else { + alert('Error: ' + data.error); + btn.disabled = false; + } + } catch (error) { + console.error(error); + btn.disabled = false; + } +} + async function deleteSchedule(id, btn) { if (!confirm('Delete this schedule?')) return;