From 9649e3c74d2783a5f3fab77276e0a611bc7864b4 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 28 Jan 2026 10:51:18 -0700 Subject: [PATCH 1/5] Add encoder health check to detect and kill stuck processes Monitors the encoder pipeline for hung ab-av1/ffmpeg processes that produce no progress. Warns at 30 minutes, automatically kills the stuck process at 1 hour to allow the pipeline to continue. Co-Authored-By: Claude Opus 4.5 --- lib/reencodarr/application.ex | 3 +- lib/reencodarr/encoder/health_check.ex | 138 +++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 lib/reencodarr/encoder/health_check.ex diff --git a/lib/reencodarr/application.ex b/lib/reencodarr/application.ex index 089bf435..aa81f059 100644 --- a/lib/reencodarr/application.ex +++ b/lib/reencodarr/application.ex @@ -55,7 +55,8 @@ defmodule Reencodarr.Application do broadway_workers = [ Reencodarr.AbAv1, Reencodarr.CrfSearcher.Supervisor, - Reencodarr.Encoder.Supervisor + Reencodarr.Encoder.Supervisor, + Reencodarr.Encoder.HealthCheck ] # Only start Analyzer GenStage in non-test environments diff --git a/lib/reencodarr/encoder/health_check.ex b/lib/reencodarr/encoder/health_check.ex new file mode 100644 index 00000000..869ceae5 --- /dev/null +++ b/lib/reencodarr/encoder/health_check.ex @@ -0,0 +1,138 @@ +defmodule Reencodarr.Encoder.HealthCheck do + @moduledoc """ + Monitors the encoder pipeline for stuck states. + + Detects when ab-av1/ffmpeg hangs silently (port alive but no progress) + and automatically kills the stuck process after 1 hour. + """ + + use GenServer + require Logger + + alias Reencodarr.Dashboard.Events + + # Check every 60 seconds + @check_interval 60_000 + # Warn at 30 minutes + @warn_threshold 30 * 60_000 + # Kill at 1 hour + @kill_threshold 60 * 60_000 + + def start_link(opts), do: GenServer.start_link(__MODULE__, opts, name: __MODULE__) + + @impl true + def init(_opts) do + schedule_check() + {:ok, %{last_progress_time: nil, last_progress_percent: nil, warned: false}} + end + + @impl true + def handle_info(:health_check, state) do + new_state = perform_health_check(state) + schedule_check() + {:noreply, new_state} + end + + defp perform_health_check(state) do + case get_encode_state() do + {:ok, encode_state} -> + check_encoder(encode_state, state) + + :error -> + state + end + end + + defp get_encode_state do + case GenServer.whereis(Reencodarr.AbAv1.Encode) do + nil -> + :error + + pid -> + try do + {:ok, :sys.get_state(pid)} + catch + :exit, _ -> :error + end + end + end + + defp check_encoder(%{port: :none}, state) do + # Not encoding, reset state + %{state | last_progress_time: nil, last_progress_percent: nil, warned: false} + end + + defp check_encoder(encode_state, state) do + now = System.monotonic_time(:millisecond) + current_percent = encode_state.last_progress && encode_state.last_progress.percent + + cond do + # Progress changed - reset timer + current_percent != state.last_progress_percent -> + %{state | last_progress_time: now, last_progress_percent: current_percent, warned: false} + + # No progress time yet (just started tracking) + state.last_progress_time == nil -> + %{state | last_progress_time: now} + + # Stuck for 1 hour - kill the process + now - state.last_progress_time > @kill_threshold -> + kill_stuck_encoder(encode_state) + %{state | last_progress_time: nil, last_progress_percent: nil, warned: false} + + # Stuck for 30 min - warn + now - state.last_progress_time > @warn_threshold and not state.warned -> + warn_stuck_encoder(encode_state) + %{state | warned: true} + + true -> + state + end + end + + defp warn_stuck_encoder(encode_state) do + video_id = encode_state.video != :none && encode_state.video.id + video_path = encode_state.video != :none && encode_state.video.path + + Logger.warning( + "Encoder may be stuck - no progress for 30+ minutes. " <> + "Video ID: #{video_id}, Path: #{video_path}" + ) + + Events.broadcast_event(:encoder_health_alert, %{ + reason: :stalled_30_min, + video_id: video_id, + video_path: video_path + }) + end + + defp kill_stuck_encoder(encode_state) do + video_id = encode_state.video != :none && encode_state.video.id + video_path = encode_state.video != :none && encode_state.video.path + + case Port.info(encode_state.port, :os_pid) do + {:os_pid, pid} -> + Logger.error( + "Killing stuck encoder process (PID: #{pid}) after 1 hour of no progress. " <> + "Video ID: #{video_id}, Path: #{video_path}" + ) + + System.cmd("kill", [to_string(pid)]) + + Events.broadcast_event(:encoder_health_alert, %{ + reason: :killed_stuck_process, + video_id: video_id, + video_path: video_path, + os_pid: pid + }) + + _ -> + Logger.error( + "Could not get OS PID for stuck encoder port. " <> + "Video ID: #{video_id}, Path: #{video_path}" + ) + end + end + + defp schedule_check, do: Process.send_after(self(), :health_check, @check_interval) +end From 90a2ccd9ffae3b6c2f0ed3fbd0d64a9d257424b7 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 28 Jan 2026 10:52:53 -0700 Subject: [PATCH 2/5] Display encoder health alerts on dashboard Shows flash error notifications when the encoder health check detects a stalled encoder (30 min warning) or kills a stuck process (1 hour). Co-Authored-By: Claude Opus 4.5 --- lib/reencodarr_web/live/dashboard_live.ex | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index 8b7214c1..d7581edf 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -195,6 +195,24 @@ defmodule ReencodarrWeb.DashboardLive do {:noreply, assign(socket, :analyzer_progress, progress)} end + # Encoder health alert handler + @impl true + def handle_info({:encoder_health_alert, data}, socket) do + message = + case data.reason do + :stalled_30_min -> + "Encoder may be stuck - no progress for 30+ minutes (#{Path.basename(data.video_path || "unknown")})" + + :killed_stuck_process -> + "Killed stuck encoder after 1 hour (#{Path.basename(data.video_path || "unknown")})" + + _ -> + "Encoder health alert: #{inspect(data.reason)}" + end + + {:noreply, put_flash(socket, :error, message)} + end + # Completion and reset handlers @impl true def handle_info({:encoding_completed, _data}, socket) do From 398ac13626b8ee4c293db5a9c1c9d6c61d94ecde Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 28 Jan 2026 11:18:45 -0700 Subject: [PATCH 3/5] Refactor encoder pipeline for simpler error handling - PostProcessor: Always attempt Sonarr notification even if DB operations fail (fixes silent failure where encode succeeded but Sonarr was never notified) - encode.ex: Replace 85-line manual retry loop with Core.Retry (exponential backoff with jitter, max 5 attempts, then move on) - health_check.ex: Switch to event-based monitoring via PubSub instead of polling encoder state with :sys.get_state(). Only accesses encoder state when killing a stuck process. Co-Authored-By: Claude Opus 4.5 --- lib/reencodarr/ab_av1/encode.ex | 113 +++----------- lib/reencodarr/encoder/health_check.ex | 197 +++++++++++++++---------- lib/reencodarr/post_processor.ex | 16 +- 3 files changed, 156 insertions(+), 170 deletions(-) diff --git a/lib/reencodarr/ab_av1/encode.ex b/lib/reencodarr/ab_av1/encode.ex index 842267af..c5e8c9b8 100644 --- a/lib/reencodarr/ab_av1/encode.ex +++ b/lib/reencodarr/ab_av1/encode.ex @@ -10,6 +10,7 @@ defmodule Reencodarr.AbAv1.Encode do alias Reencodarr.AbAv1.Helper alias Reencodarr.AbAv1.ProgressParser + alias Reencodarr.Core.Retry alias Reencodarr.Dashboard.Events alias Reencodarr.Encoder.Broadway.Producer alias Reencodarr.{Media, PostProcessor} @@ -120,14 +121,8 @@ defmodule Reencodarr.AbAv1.Encode do {port, {:exit_status, exit_code}}, %{port: port, vmaf: vmaf, output_file: output_file} = state ) do - result = - case exit_code do - 0 -> {:ok, :success} - _ -> {:error, exit_code} - end - # Broadcast encoding completion to Dashboard Events - pubsub_result = if result == {:ok, :success}, do: :success, else: {:error, exit_code} + pubsub_result = if exit_code == 0, do: :success, else: {:error, exit_code} Events.broadcast_event(:encoding_completed, %{ video_id: vmaf.video.id, @@ -137,91 +132,17 @@ defmodule Reencodarr.AbAv1.Encode do # Notify the Broadway producer that encoding is now available Producer.dispatch_available() - # Attempt to notify success/failure, but don't crash if DB is busy — retry later - notify_result = - try do - if result == {:ok, :success} do - notify_encoder_success(vmaf.video, output_file) - else - notify_encoder_failure(vmaf.video, exit_code) - end - - :ok - rescue - e in Exqlite.Error -> - Logger.warning( - "Encode: Database busy while handling exit_status, scheduling retry: #{inspect(e)}" - ) - - # Retry after short backoff. Keep current state until retry completes. - Process.send_after(self(), {:encoding_exit_retry, exit_code, vmaf, output_file}, 200) - :retry_scheduled - end - - case notify_result do - :ok -> - new_state = %{ - state - | port: :none, - video: :none, - vmaf: :none, - output_file: nil, - partial_line_buffer: "", - last_progress: nil - } - - {:noreply, new_state} - - :retry_scheduled -> - # Keep state intact so retry handler has data available - {:noreply, state} - end - end - - @impl true - def handle_info({:encoding_exit_retry, exit_code, vmaf, output_file}, state) do - Logger.info("Encode: retrying exit_status handling for VMAF #{vmaf.id}") - - retry_result = - try do - if exit_code == 0 do - notify_encoder_success(vmaf.video, output_file) - else - notify_encoder_failure(vmaf.video, exit_code) - end - - :ok - rescue - e in Exqlite.Error -> - Logger.warning( - "Encode: retry failed due to DB busy: #{inspect(e)}, scheduling another retry" - ) - - Process.send_after(self(), {:encoding_exit_retry, exit_code, vmaf, output_file}, 500) - :retry_scheduled + # Handle success/failure with automatic retry on DB busy + Retry.retry_on_db_busy(fn -> + if exit_code == 0 do + notify_encoder_success(vmaf.video, output_file) + else + notify_encoder_failure(vmaf.video, exit_code) end + end) - case retry_result do - :ok -> - # Clean up state now that notification succeeded - cleared_state = %{ - state - | port: :none, - video: :none, - vmaf: :none, - output_file: nil, - partial_line_buffer: "", - last_progress: nil - } - - # Ensure Broadway knows encoder is available - Producer.dispatch_available() - - {:noreply, cleared_state} - - :retry_scheduled -> - {:noreply, state} - end + # Always clear state after handling exit (retry succeeded or max attempts reached) + {:noreply, clear_state(state)} end # Catch-all for any other port messages @@ -349,6 +270,18 @@ defmodule Reencodarr.AbAv1.Encode do def build_encode_args_for_test(vmaf), do: build_encode_args(vmaf) end + defp clear_state(state) do + %{ + state + | port: :none, + video: :none, + vmaf: :none, + output_file: nil, + partial_line_buffer: "", + last_progress: nil + } + end + defp notify_encoder_success(video, output_file) do # Use PostProcessor for cleanup work PostProcessor.process_encoding_success(video, output_file) diff --git a/lib/reencodarr/encoder/health_check.ex b/lib/reencodarr/encoder/health_check.ex index 869ceae5..3b719e7d 100644 --- a/lib/reencodarr/encoder/health_check.ex +++ b/lib/reencodarr/encoder/health_check.ex @@ -1,9 +1,10 @@ defmodule Reencodarr.Encoder.HealthCheck do @moduledoc """ - Monitors the encoder pipeline for stuck states. + Monitors the encoder pipeline for stuck states via PubSub events. - Detects when ab-av1/ffmpeg hangs silently (port alive but no progress) - and automatically kills the stuck process after 1 hour. + Subscribes to encoding events and detects when ab-av1/ffmpeg hangs silently + (no progress events for extended periods). Automatically kills the stuck + process after 1 hour of no progress. """ use GenServer @@ -22,10 +23,67 @@ defmodule Reencodarr.Encoder.HealthCheck do @impl true def init(_opts) do + # Subscribe to encoding events instead of polling state + Phoenix.PubSub.subscribe(Reencodarr.PubSub, Events.channel()) schedule_check() - {:ok, %{last_progress_time: nil, last_progress_percent: nil, warned: false}} + + {:ok, + %{ + encoding: false, + video_id: nil, + video_path: nil, + last_progress_time: nil, + last_progress_percent: nil, + warned: false + }} + end + + # Handle encoding lifecycle events + @impl true + def handle_info({:encoding_started, data}, state) do + now = System.monotonic_time(:millisecond) + + {:noreply, + %{ + state + | encoding: true, + video_id: data[:video_id], + video_path: data[:filename], + last_progress_time: now, + last_progress_percent: nil, + warned: false + }} + end + + @impl true + def handle_info({:encoding_progress, data}, state) do + now = System.monotonic_time(:millisecond) + percent = data[:percent] + + {:noreply, + %{ + state + | last_progress_time: now, + last_progress_percent: percent, + warned: false + }} + end + + @impl true + def handle_info({:encoding_completed, _data}, state) do + {:noreply, + %{ + state + | encoding: false, + video_id: nil, + video_path: nil, + last_progress_time: nil, + last_progress_percent: nil, + warned: false + }} end + # Periodic health check @impl true def handle_info(:health_check, state) do new_state = perform_health_check(state) @@ -33,56 +91,30 @@ defmodule Reencodarr.Encoder.HealthCheck do {:noreply, new_state} end - defp perform_health_check(state) do - case get_encode_state() do - {:ok, encode_state} -> - check_encoder(encode_state, state) - - :error -> - state - end - end - - defp get_encode_state do - case GenServer.whereis(Reencodarr.AbAv1.Encode) do - nil -> - :error + # Ignore other PubSub events + @impl true + def handle_info(_msg, state), do: {:noreply, state} - pid -> - try do - {:ok, :sys.get_state(pid)} - catch - :exit, _ -> :error - end - end - end + defp perform_health_check(%{encoding: false} = state), do: state - defp check_encoder(%{port: :none}, state) do - # Not encoding, reset state - %{state | last_progress_time: nil, last_progress_percent: nil, warned: false} + defp perform_health_check(%{encoding: true, last_progress_time: nil} = state) do + # Encoding but no progress time set - initialize it + %{state | last_progress_time: System.monotonic_time(:millisecond)} end - defp check_encoder(encode_state, state) do + defp perform_health_check(%{encoding: true} = state) do now = System.monotonic_time(:millisecond) - current_percent = encode_state.last_progress && encode_state.last_progress.percent + elapsed = now - state.last_progress_time cond do - # Progress changed - reset timer - current_percent != state.last_progress_percent -> - %{state | last_progress_time: now, last_progress_percent: current_percent, warned: false} - - # No progress time yet (just started tracking) - state.last_progress_time == nil -> - %{state | last_progress_time: now} - # Stuck for 1 hour - kill the process - now - state.last_progress_time > @kill_threshold -> - kill_stuck_encoder(encode_state) - %{state | last_progress_time: nil, last_progress_percent: nil, warned: false} + elapsed > @kill_threshold -> + kill_stuck_encoder(state) + %{state | encoding: false, last_progress_time: nil, warned: false} # Stuck for 30 min - warn - now - state.last_progress_time > @warn_threshold and not state.warned -> - warn_stuck_encoder(encode_state) + elapsed > @warn_threshold and not state.warned -> + warn_stuck_encoder(state) %{state | warned: true} true -> @@ -90,47 +122,60 @@ defmodule Reencodarr.Encoder.HealthCheck do end end - defp warn_stuck_encoder(encode_state) do - video_id = encode_state.video != :none && encode_state.video.id - video_path = encode_state.video != :none && encode_state.video.path - + defp warn_stuck_encoder(state) do Logger.warning( "Encoder may be stuck - no progress for 30+ minutes. " <> - "Video ID: #{video_id}, Path: #{video_path}" + "Video ID: #{state.video_id}, Path: #{state.video_path}" ) Events.broadcast_event(:encoder_health_alert, %{ reason: :stalled_30_min, - video_id: video_id, - video_path: video_path + video_id: state.video_id, + video_path: state.video_path }) end - defp kill_stuck_encoder(encode_state) do - video_id = encode_state.video != :none && encode_state.video.id - video_path = encode_state.video != :none && encode_state.video.path - - case Port.info(encode_state.port, :os_pid) do - {:os_pid, pid} -> - Logger.error( - "Killing stuck encoder process (PID: #{pid}) after 1 hour of no progress. " <> - "Video ID: #{video_id}, Path: #{video_path}" - ) - - System.cmd("kill", [to_string(pid)]) - - Events.broadcast_event(:encoder_health_alert, %{ - reason: :killed_stuck_process, - video_id: video_id, - video_path: video_path, - os_pid: pid - }) - - _ -> - Logger.error( - "Could not get OS PID for stuck encoder port. " <> - "Video ID: #{video_id}, Path: #{video_path}" - ) + defp kill_stuck_encoder(state) do + # Only place we need to access encoder state - to get the port for killing + case get_encoder_port() do + {:ok, port} -> + case Port.info(port, :os_pid) do + {:os_pid, pid} -> + Logger.error( + "Killing stuck encoder process (PID: #{pid}) after 1 hour of no progress. " <> + "Video ID: #{state.video_id}, Path: #{state.video_path}" + ) + + System.cmd("kill", [to_string(pid)]) + + Events.broadcast_event(:encoder_health_alert, %{ + reason: :killed_stuck_process, + video_id: state.video_id, + video_path: state.video_path, + os_pid: pid + }) + + _ -> + Logger.error("Could not get OS PID for stuck encoder port") + end + + :error -> + Logger.error("Could not access encoder state to kill stuck process") + end + end + + defp get_encoder_port do + case GenServer.whereis(Reencodarr.AbAv1.Encode) do + nil -> + :error + + pid -> + try do + state = :sys.get_state(pid) + if state.port != :none, do: {:ok, state.port}, else: :error + catch + :exit, _ -> :error + end end end diff --git a/lib/reencodarr/post_processor.ex b/lib/reencodarr/post_processor.ex index 3b7b67a6..c0106b38 100644 --- a/lib/reencodarr/post_processor.ex +++ b/lib/reencodarr/post_processor.ex @@ -76,8 +76,12 @@ defmodule Reencodarr.PostProcessor do defp process_intermediate_success(video, actual_path) do case Repo.reload(video) do nil -> - Logger.error("Failed to reload video #{video.id}: Video not found.") - :ok + Logger.error( + "Failed to reload video #{video.id}: Video not found. Still attempting sync." + ) + + # File was moved successfully, still try to notify Sonarr with original video data + finalize_and_sync(video, actual_path) reloaded -> process_reloaded_video(reloaded, actual_path) @@ -93,8 +97,12 @@ defmodule Reencodarr.PostProcessor do finalize_and_sync(updated_video, actual_path) {:error, reason} -> - Logger.error("Failed to mark video #{video.id} as re-encoded: #{inspect(reason)}") - :ok + Logger.error( + "Failed to mark video #{video.id} as re-encoded: #{inspect(reason)}. Still attempting sync." + ) + + # DB update failed but file was moved, still try to finalize and notify Sonarr + finalize_and_sync(video, actual_path) end end From 4bcb784974657e3b42a9a9c2df87cf5b130b5fd0 Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 28 Jan 2026 11:24:44 -0700 Subject: [PATCH 4/5] Record encoding failures in FailureTracker notify_encoder_failure was a no-op; now delegates to PostProcessor.process_encoding_failure which records failures for visibility in the dashboard. Co-Authored-By: Claude Opus 4.5 --- lib/reencodarr/ab_av1/encode.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/reencodarr/ab_av1/encode.ex b/lib/reencodarr/ab_av1/encode.ex index c5e8c9b8..d9ee7fb6 100644 --- a/lib/reencodarr/ab_av1/encode.ex +++ b/lib/reencodarr/ab_av1/encode.ex @@ -287,8 +287,8 @@ defmodule Reencodarr.AbAv1.Encode do PostProcessor.process_encoding_success(video, output_file) end - defp notify_encoder_failure(_video, _exit_code) do - # Failure handling complete - no additional work needed + defp notify_encoder_failure(video, exit_code) do + PostProcessor.process_encoding_failure(video, exit_code) end # Extract progress data from a line and store it for periodic updates From fb424c447883f8e744a840dfb1abbf151509b2da Mon Sep 17 00:00:00 2001 From: Mika Cohen Date: Wed, 28 Jan 2026 12:15:06 -0700 Subject: [PATCH 5/5] Address PR review comments for encoder health check - Reset all state fields when killing stuck encoder (not just encoding flag) - Replace :sys.get_state anti-pattern with proper PubSub approach: - Encode broadcasts os_pid when encoding starts - HealthCheck tracks os_pid from events instead of introspecting state - Use Task.start for async kill command to avoid blocking GenServer - Fix Path.basename(nil) error in dashboard health alert handler - Add clarifying comment about why failures don't notify Sonarr Co-Authored-By: Claude Opus 4.5 --- lib/reencodarr/ab_av1/encode.ex | 12 +++- lib/reencodarr/encoder/health_check.ex | 81 +++++++++++------------ lib/reencodarr/post_processor.ex | 2 + lib/reencodarr_web/live/dashboard_live.ex | 6 +- 4 files changed, 53 insertions(+), 48 deletions(-) diff --git a/lib/reencodarr/ab_av1/encode.ex b/lib/reencodarr/ab_av1/encode.ex index d9ee7fb6..5bed3469 100644 --- a/lib/reencodarr/ab_av1/encode.ex +++ b/lib/reencodarr/ab_av1/encode.ex @@ -213,10 +213,18 @@ defmodule Reencodarr.AbAv1.Encode do port = Helper.open_port(args) - # Broadcast encoding started to Dashboard Events + # Get OS PID from port for health monitoring + os_pid = + case Port.info(port, :os_pid) do + {:os_pid, pid} -> pid + _ -> nil + end + + # Broadcast encoding started to Dashboard Events (includes OS PID for health check) Events.broadcast_event(:encoding_started, %{ video_id: vmaf.video.id, - filename: Path.basename(vmaf.video.path) + filename: Path.basename(vmaf.video.path), + os_pid: os_pid }) # Set up a periodic timer to check if we're still alive and potentially emit progress diff --git a/lib/reencodarr/encoder/health_check.ex b/lib/reencodarr/encoder/health_check.ex index 3b719e7d..161c4ae4 100644 --- a/lib/reencodarr/encoder/health_check.ex +++ b/lib/reencodarr/encoder/health_check.ex @@ -34,7 +34,8 @@ defmodule Reencodarr.Encoder.HealthCheck do video_path: nil, last_progress_time: nil, last_progress_percent: nil, - warned: false + warned: false, + os_pid: nil }} end @@ -51,7 +52,8 @@ defmodule Reencodarr.Encoder.HealthCheck do video_path: data[:filename], last_progress_time: now, last_progress_percent: nil, - warned: false + warned: false, + os_pid: data[:os_pid] }} end @@ -79,7 +81,8 @@ defmodule Reencodarr.Encoder.HealthCheck do video_path: nil, last_progress_time: nil, last_progress_percent: nil, - warned: false + warned: false, + os_pid: nil }} end @@ -110,7 +113,17 @@ defmodule Reencodarr.Encoder.HealthCheck do # Stuck for 1 hour - kill the process elapsed > @kill_threshold -> kill_stuck_encoder(state) - %{state | encoding: false, last_progress_time: nil, warned: false} + + %{ + state + | encoding: false, + video_id: nil, + video_path: nil, + last_progress_time: nil, + last_progress_percent: nil, + warned: false, + os_pid: nil + } # Stuck for 30 min - warn elapsed > @warn_threshold and not state.warned -> @@ -135,48 +148,28 @@ defmodule Reencodarr.Encoder.HealthCheck do }) end - defp kill_stuck_encoder(state) do - # Only place we need to access encoder state - to get the port for killing - case get_encoder_port() do - {:ok, port} -> - case Port.info(port, :os_pid) do - {:os_pid, pid} -> - Logger.error( - "Killing stuck encoder process (PID: #{pid}) after 1 hour of no progress. " <> - "Video ID: #{state.video_id}, Path: #{state.video_path}" - ) - - System.cmd("kill", [to_string(pid)]) - - Events.broadcast_event(:encoder_health_alert, %{ - reason: :killed_stuck_process, - video_id: state.video_id, - video_path: state.video_path, - os_pid: pid - }) - - _ -> - Logger.error("Could not get OS PID for stuck encoder port") - end - - :error -> - Logger.error("Could not access encoder state to kill stuck process") - end + defp kill_stuck_encoder(%{os_pid: nil} = state) do + Logger.error( + "Could not kill stuck encoder - no OS PID available. " <> + "Video ID: #{state.video_id}, Path: #{state.video_path}" + ) end - defp get_encoder_port do - case GenServer.whereis(Reencodarr.AbAv1.Encode) do - nil -> - :error - - pid -> - try do - state = :sys.get_state(pid) - if state.port != :none, do: {:ok, state.port}, else: :error - catch - :exit, _ -> :error - end - end + defp kill_stuck_encoder(%{os_pid: os_pid} = state) do + Logger.error( + "Killing stuck encoder process (PID: #{os_pid}) after 1 hour of no progress. " <> + "Video ID: #{state.video_id}, Path: #{state.video_path}" + ) + + # Run the kill command asynchronously so the HealthCheck GenServer is not blocked + Task.start(fn -> System.cmd("kill", [to_string(os_pid)]) end) + + Events.broadcast_event(:encoder_health_alert, %{ + reason: :killed_stuck_process, + video_id: state.video_id, + video_path: state.video_path, + os_pid: os_pid + }) end defp schedule_check, do: Process.send_after(self(), :health_check, @check_interval) diff --git a/lib/reencodarr/post_processor.ex b/lib/reencodarr/post_processor.ex index c0106b38..c2ca2e6b 100644 --- a/lib/reencodarr/post_processor.ex +++ b/lib/reencodarr/post_processor.ex @@ -36,6 +36,8 @@ defmodule Reencodarr.PostProcessor do ) # Record detailed failure information with enhanced context + # Note: We don't notify Sonarr/Radarr on failure since the original file is unchanged. + # Sync notification is only needed when the file has been successfully replaced. Reencodarr.FailureTracker.record_process_failure(video, exit_code, context: context) :ok diff --git a/lib/reencodarr_web/live/dashboard_live.ex b/lib/reencodarr_web/live/dashboard_live.ex index d7581edf..270dc948 100644 --- a/lib/reencodarr_web/live/dashboard_live.ex +++ b/lib/reencodarr_web/live/dashboard_live.ex @@ -198,13 +198,15 @@ defmodule ReencodarrWeb.DashboardLive do # Encoder health alert handler @impl true def handle_info({:encoder_health_alert, data}, socket) do + filename = if data.video_path, do: Path.basename(data.video_path), else: "unknown" + message = case data.reason do :stalled_30_min -> - "Encoder may be stuck - no progress for 30+ minutes (#{Path.basename(data.video_path || "unknown")})" + "Encoder may be stuck - no progress for 30+ minutes (#{filename})" :killed_stuck_process -> - "Killed stuck encoder after 1 hour (#{Path.basename(data.video_path || "unknown")})" + "Killed stuck encoder after 1 hour (#{filename})" _ -> "Encoder health alert: #{inspect(data.reason)}"