🚀 Analyzer Performance Optimizations & Queue UI Fixes

.github/copilot-instructions.md

@@ -30,6 +30,8 @@ Key pattern: Each pipeline has a Producer that checks GenServer availability bef
 
 ## Development Workflows
 
+**Command Execution Note**: All commands should be executed directly without prefixing with `cd` to the project root. The working directory is always assumed to be the project root directory.
+
 ### Essential Commands
 ```bash
 # Setup (no longer requires PostgreSQL)

.github/workflows/elixir.yml

@@ -26,7 +26,7 @@ jobs:
         echo "https://dl-cdn.alpinelinux.org/alpine/v3.21/community" >> /etc/apk/repositories
         apk update
 
-        # Install basic system dependencies
+        # Install basic system dependencies including GNU tar for GitHub Actions caching
         apk add --no-cache \
           git \
           bash \
@@ -37,7 +37,13 @@ jobs:
           zlib-dev \
           mediainfo \
           cmake \
-          make
+          make \
+          gzip \
+          tar
+
+        # Verify tar installation for GitHub Actions caching compatibility
+        # GitHub Actions cache requires POSIX-compliant tar
+        which tar && tar --version | head -1
 
         # Install FFmpeg with essential codec libraries (minimal set to avoid conflicts)
         apk add --no-cache \

.iex.exs

@@ -1,12 +1,119 @@
 IEx.configure(auto_reload: true)
 
-alias Reencodarr.AbAv1
-alias Reencodarr.Media
+# Core modules
+alias Reencodarr.{Media, Repo, Rules, Services, Sync, Config}
 alias Reencodarr.Media.{Video, Library, Vmaf}
-alias Reencodarr.Repo
-alias Reencodarr.Rules
-alias Reencodarr.{Analyzer, ManualScanner}
-alias Reencodarr.Services
-alias Reencodarr.Sync
+
+# Broadway pipelines and workers
+alias Reencodarr.{Analyzer, CrfSearcher, Encoder}
+alias Reencodarr.Analyzer.{Broadway, QueueManager}
+alias Reencodarr.AbAv1.{CrfSearch, Encode}
+
+# State management
+alias Reencodarr.PipelineStateMachine
+
+# Dashboard and events
+alias Reencodarr.Dashboard.Events
+alias ReencodarrWeb.{Endpoint, Router}
 
 import Ecto.Query
+
+# Utility functions for debugging
+defmodule IExHelpers do
+  @doc "Show status of all Broadway pipelines"
+  def pipelines_status do
+    %{
+      analyzer: Analyzer.status(),
+      crf_searcher: CrfSearcher.status(),
+      encoder: Encoder.status()
+    }
+  end
+
+  @doc "Get count of items in each queue"
+  def queue_counts do
+    %{
+      analysis: Analyzer.queue_count(),
+      crf_search: CrfSearcher.queue_count(),
+      encoding: Encoder.queue_count()
+    }
+  end
+
+  @doc "Get next items in each queue"
+  def next_items(limit \\ 5) do
+    %{
+      analysis: Analyzer.next_videos(limit),
+      crf_search: CrfSearcher.next_videos(limit),
+      encoding: Encoder.next_videos(limit)
+    }
+  end
+
+  @doc "Start all pipelines"
+  def start_all do
+    Analyzer.start()
+    CrfSearcher.start()
+    Encoder.start()
+    pipelines_status()
+  end
+
+  @doc "Pause all pipelines"
+  def pause_all do
+    Analyzer.pause()
+    CrfSearcher.pause()
+    Encoder.pause()
+    pipelines_status()
+  end
+
+  @doc "Get video by path (fuzzy match)"
+  def find_video(path_fragment) do
+    from(v in Video, where: ilike(v.path, ^"%#{path_fragment}%"))
+    |> Repo.all()
+  end
+
+  @doc "Get service configs"
+  def configs do
+    Services.list_services()
+  end
+
+  @doc "Quick video state summary"
+  def video_states do
+    from(v in Video,
+      group_by: v.state,
+      select: {v.state, count()}
+    )
+    |> Repo.all()
+    |> Enum.into(%{})
+  end
+
+  @doc "Debug a specific video"
+  def debug_video(id) when is_integer(id) do
+    video = Repo.get!(Video, id) |> Repo.preload(:vmafs)
+
+    %{
+      video: video,
+      vmaf_count: length(video.vmafs),
+      chosen_vmaf: Enum.find(video.vmafs, & &1.chosen),
+      service: Services.get_service_by_id(video.service_id)
+    }
+  end
+end
+
+# Import the helper functions
+import IExHelpers
+
+# Welcome message
+IO.puts("""
+
+🚀 Reencodarr IEx Console Ready!
+
+Quick commands:
+  pipelines_status()  - Check all Broadway pipeline status
+  queue_counts()      - Get queue counts for all pipelines
+  next_items()        - See next items in each queue
+  start_all() / pause_all() - Control all pipelines
+  video_states()      - Summary of video states
+  find_video("path")  - Find videos by path fragment
+  debug_video(123)    - Debug specific video by ID
+  configs()           - List service configurations
+
+Happy debugging! 🎬
+""")

config/dev.exs

@@ -100,19 +100,5 @@ config :phoenix_live_view,
 # Disable swoosh api client as it is only required for production adapters.
 config :swoosh, :api_client, false
 
-config :live_debugger,
-  # IP on which LiveDebugger will be hosted
-  ip: {127, 0, 0, 1},
-  # Port on which LiveDebugger will be hosted
-  port: 4007,
-  # Secret key used for LiveDebugger.Endpoint
-  secret_key_base: "nxeC3Z+0GcRf0NKbjgCuwBQvld+KftsF+Ebsti02FZUXc6xQNR80V8Kdw5sEYiEk",
-  # Signing salt used for LiveDebugger.Endpoint
-  signing_salt: "QvcMLz2F3V0jO1LRee/B5207o3KoSJoMkdKP/c/V2rt4x2hxZmg7UF74uzN7QAbP",
-  # Adapter used in LiveDebugger.Endpoint
-  adapter: Bandit.PhoenixAdapter,
-  # Forces LiveDebugger to start even if project is not started with the `mix phx.server`
-  server: true
-
 # Load local overrides from dev.overrides.exs if exists
 if File.exists?("config/dev.overrides.exs"), do: import_config("dev.overrides.exs")

config/test.exs

@@ -10,8 +10,11 @@ config :reencodarr, Reencodarr.Repo,
   pool: Ecto.Adapters.SQL.Sandbox,
   # Use single connection for test sandbox
   pool_size: 1,
-  # Test-specific timeout
-  timeout: 30_000
+  # Test-specific timeout for query execution
+  timeout: 30_000,
+  # Pool checkout timeout settings for handling concurrent test access
+  queue_target: 5_000,
+  queue_interval: 10_000
 
 # We don't run a server during test. If one is required,
 # you can enable the server option below.

lib/reencodarr/ab_av1.ex

@@ -85,7 +85,6 @@ defmodule Reencodarr.AbAv1 do
   @doc false
   def init(:ok) do
     children = [
-      Reencodarr.AbAv1.CrfSearch,
       Reencodarr.AbAv1.Encode
     ]