Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 101 additions & 34 deletions .github/workflows/elixir.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,66 +11,133 @@ permissions:

jobs:
build:

name: Build and test
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
container:
image: alpine:3.21

env:
MIX_ENV: test

steps:
- name: Install system dependencies and FFmpeg
run: |
# Add community repository for 3.21 packages
echo "https://dl-cdn.alpinelinux.org/alpine/v3.21/community" >> /etc/apk/repositories
apk update

# Install basic system dependencies
apk add --no-cache \
git \
bash \
curl \
build-base \
openssl-dev \
ncurses-dev \
zlib-dev \
mediainfo \
cmake \
make

# Install FFmpeg with essential codec libraries (minimal set to avoid conflicts)
apk add --no-cache \
ffmpeg \
x264-libs \
x265-libs \
svt-av1

- name: Install Erlang and Elixir
run: |
# Install Erlang 27 and Elixir 1.19 from Alpine 3.21 (should be available in community repo)
apk add --no-cache \
erlang \
elixir

- uses: actions/checkout@v4

- name: Set up Elixir
uses: erlef/setup-beam@61e01a43a562a89bfc54c7f9a378ff67b03e4a21 # v1.16.0
with:
elixir-version: '1.18.3' # [Required] Define the Elixir version
otp-version: '27.3.4' # [Required] Define the Erlang/OTP version

- name: Install and cache apt packages
uses: awalsh128/cache-apt-pkgs-action@v1
- name: Cache rustup and cargo installation
uses: actions/cache@v4
with:
packages: fd-find ffmpeg zstd mediainfo
version: 1.0
path: |
~/.rustup
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git
key: ${{ runner.os }}-alpine-rust-${{ hashFiles('**/Cargo.lock') }}-v1
restore-keys: |
${{ runner.os }}-alpine-rust-

- name: Cache ab-av1 binary
id: cache-ab-av1
uses: actions/cache@v4
id: ab-av1-cache
with:
path: /usr/local/bin/ab-av1
key: ab-av1-v0.10.1

path: ~/.cargo/bin/ab-av1
key: ${{ runner.os }}-alpine-ab-av1-${{ hashFiles('~/.cargo/bin/ab-av1') }}-v1
restore-keys: |
${{ runner.os }}-alpine-ab-av1-

- name: Install ab-av1
if: steps.cache-ab-av1.outputs.cache-hit != 'true'
if: steps.ab-av1-cache.outputs.cache-hit != 'true'
run: |
# Install rustup to get latest stable Rust (Alpine's is too old)
if [ ! -f ~/.cargo/env ]; then
apk add --no-cache curl
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
fi
source ~/.cargo/env
# Add cargo to PATH for this and future steps
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
# Install latest stable ab-av1 with newer Rust toolchain
cargo install ab-av1

- name: Setup cargo PATH
run: |
# Install ab-av1 from GitHub releases
wget -O /tmp/ab-av1.tar.zst https://github.com/alexheretic/ab-av1/releases/download/v0.10.1/ab-av1-v0.10.1-x86_64-unknown-linux-musl.tar.zst
cd /tmp && zstd -d ab-av1.tar.zst && tar -xf ab-av1.tar
sudo mv ab-av1 /usr/local/bin/ab-av1
sudo chmod +x /usr/local/bin/ab-av1
# Ensure cargo is available in PATH for subsequent steps
echo "$HOME/.cargo/bin" >> $GITHUB_PATH

- name: Restore dependencies cache
- name: Cache Mix dependencies
uses: actions/cache@v4
with:
path: deps
key: ${{ runner.os }}-mix-deps-${{ hashFiles('**/mix.lock') }}
restore-keys: ${{ runner.os }}-mix-deps-
path: |
deps
_build/test/lib
key: ${{ runner.os }}-alpine-mix-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-alpine-mix-

- name: Restore build cache
- name: Cache compiled dependencies
uses: actions/cache@v4
with:
path: _build
key: ${{ runner.os }}-mix-build-${{ hashFiles('**/mix.lock') }}-${{ hashFiles('**/*.ex') }}-${{ hashFiles('**/*.exs') }}
key: ${{ runner.os }}-alpine-build-${{ hashFiles('mix.lock') }}-${{ hashFiles('lib/**/*.ex', 'test/**/*.exs', 'config/**/*.exs') }}
restore-keys: |
${{ runner.os }}-mix-build-${{ hashFiles('**/mix.lock') }}-
${{ runner.os }}-mix-build-
${{ runner.os }}-alpine-build-${{ hashFiles('mix.lock') }}-
${{ runner.os }}-alpine-build-

- name: Cache native dependencies
uses: actions/cache@v4
with:
path: |
deps/*/priv
deps/*/_build
key: ${{ runner.os }}-alpine-native-${{ hashFiles('mix.lock') }}
restore-keys: |
${{ runner.os }}-alpine-native-

- name: Verify environment
run: |
echo 'Elixir version:' && elixir --version
echo 'Erlang version:' && erl -eval 'erlang:display(erlang:system_info(otp_release)), halt().' -noshell
echo 'ab-av1 version:' && ab-av1 --version
echo 'mediainfo version:' && mediainfo --version | head -1
echo 'ffmpeg version:' && ffmpeg -version | head -1

- name: Install dependencies
run: mix deps.get

- name: Compile dependencies
- name: Compile dependencies
run: mix deps.compile

- name: Set MIX_ENV to test
run: echo "MIX_ENV=test" >> $GITHUB_ENV

- name: Compile project
run: mix compile --warnings-as-errors

Expand Down
24 changes: 11 additions & 13 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,8 @@
system: let
pkgs = import nixpkgs {inherit system;};
lib = pkgs.lib;
# current is 28.0.2
erlang = pkgs.erlang;
# erlang = pkgs.erlang.override {
# version = "28.0.2";
# src = pkgs.fetchurl {
# url = "https://github.com/erlang/otp/releases/download/OTP-${erlang.version}/otp_src_${erlang.version}.tar.gz";
# sha256 = "sha256-zkPciimta8G22/yX8FPS6FC0pMKQ7KBlBY1rM85HbbU=";
# };
# };
# Use latest OTP 28.1 with Elixir 1.19.0-rc.0 for cutting-edge features
erlang = pkgs.erlang_28;
beamPackages = pkgs.beam.packagesWith erlang;
elixir = beamPackages.elixir.override {
erlang = erlang;
Expand Down Expand Up @@ -67,7 +60,7 @@
nativeBuildInputs =
[
erlang
pkgs.elixir
elixir
beamPackages.ex_doc
beamPackages.hex
beamPackages.rebar
Expand All @@ -86,6 +79,11 @@
pkgs.gnupg
pkgs.pinentry
pkgs.pinentry-curses
# Video processing tools for CI/dev
pkgs.ab-av1
pkgs.mediainfo
# GitHub Actions local testing
pkgs.act
]
++ lib.optional pkgs.stdenv.isLinux pkgs.libnotify
++ lib.optional pkgs.stdenv.isLinux pkgs.inotify-tools
Expand All @@ -100,14 +98,14 @@
export DATABASE_URL="ecto://mjc@localhost:5432/reencodarr_dev"
export SECRET_KEY_BASE="WEWsPGIpK/OgJA2ZcwzsgZxWKSAp35IsqWPYsvSUmm5awBUGpvsVOcG2kkDteXR1"
export COMPOSE_BAKE=true

# GPG Configuration
export GPG_TTY=$(tty)
export PINENTRY_USER_DATA="USE_CURSES=1"

# Ensure GPG agent is using the right pinentry
echo "pinentry-program ${pkgs.pinentry-curses}/bin/pinentry-curses" >> ~/.gnupg/gpg-agent.conf 2>/dev/null || true

# Configure git to use nix-provided GPG
git config --global gpg.program "${pkgs.gnupg}/bin/gpg"
'';
Expand Down
78 changes: 50 additions & 28 deletions lib/reencodarr/ab_av1/output_parser.ex
Original file line number Diff line number Diff line change
Expand Up @@ -8,38 +8,56 @@ defmodule Reencodarr.AbAv1.OutputParser do

alias Reencodarr.Core.Parsers

# Pattern definitions
@patterns %{
encoding_sample:
~r/encoding\ssample\s(?<sample_num>\d+)\/(?<total_samples>\d+)\scrf\s(?<crf>\d+(?:\.\d+)?)/,
simple_vmaf:
~r/(?:\[(?<timestamp>[^\]]+)\].*?)?crf\s(?<crf>\d+(?:\.\d+)?)\sVMAF\s(?<score>\d+\.\d+)\s\((?<percent>\d+)%\)/,
sample_vmaf:
~r/sample\s(?<sample_num>\d+)\/(?<total_samples>\d+)\scrf\s(?<crf>\d+(?:\.\d+)?)\sVMAF\s(?<score>\d+\.\d+)\s\((?<percent>\d+)%\)/,
dash_vmaf: ~r/^-\scrf\s(?<crf>\d+(?:\.\d+)?)\sVMAF\s(?<score>\d+\.\d+)\s\((?<percent>\d+)%\)/,
eta_vmaf:
~r/crf\s(?<crf>\d+(?:\.\d+)?)\sVMAF\s(?<score>\d+\.\d+)\spredicted\svideo\sstream\ssize\s(?<size>\d+\.?\d*)\s(?<unit>\w+)\s\((?<percent>\d+)%\)\staking\s(?<time>\d+\.?\d*)\s(?<time_unit>\w+)/,
vmaf_comparison: ~r/vmaf\s(?<file1>.+?)\svs\sreference\s(?<file2>.+)/,
progress:
~r/\[(?<timestamp>[^\]]+)\].*?(?<progress>\d+(?:\.\d+)?)%,\s(?<fps>\d+(?:\.\d+)?)\sfps?,\seta\s(?<eta>\d+)\s(?<time_unit>second|minute|hour|day|week|month|year)s?/,
success: ~r/(?:\[.*\]\s)?crf\s(?<crf>\d+(?:\.\d+)?)\ssuccessful/,
warning: ~r/^Warning:\s(?<message>.*)/,
encoding_start: ~r/\[.*\] encoding (?<filename>\d+\.mkv)/,
encoding_progress:
~r/\[.*\]\s*(?<percent>\d+)%,\s*(?<fps>[\d\.]+)\s*fps,\s*eta\s*(?<eta>\d+)\s*(?<unit>minutes|seconds|hours|days|weeks|months|years)/,
encoding_progress_alt:
~r/(?<percent>\d+)%,\s*(?<fps>[\d\.]+)\s*fps,\s*eta\s*(?<eta>\d+)\s*(?<unit>minutes|seconds|hours|days|weeks|months|years)/,
file_size_progress: ~r/Encoded\s(?<size>[\d\.]+\s\w+)\s\((?<percent>\d+)%\)/,
ffmpeg_error: ~r/Error: ffmpeg encode exit code (?<exit_code>\d+)/
}
@doc """
Returns the centralized regex patterns for ab-av1 output parsing.

This function provides access to the regex patterns for other modules
that need to do pattern matching without duplicating the definitions.

Note: Patterns are compiled fresh on each call for simplicity and Elixir 1.19
compatibility. This trades a small performance cost for cleaner, more maintainable code.
"""
@spec get_patterns() :: map()
def get_patterns, do: @patterns
def get_patterns do
%{
encoding_sample:
~r/encoding\ssample\s(?<sample_num>\d+)\/(?<total_samples>\d+)\scrf\s(?<crf>\d+(?:\.\d+)?)/,
simple_vmaf:
~r/(?:\[(?<timestamp>[^\]]+)\].*?)?crf\s(?<crf>\d+(?:\.\d+)?)\sVMAF\s(?<score>\d+\.\d+)\s\((?<percent>\d+)%\)/,
sample_vmaf:
~r/sample\s(?<sample_num>\d+)\/(?<total_samples>\d+)\scrf\s(?<crf>\d+(?:\.\d+)?)\sVMAF\s(?<score>\d+\.\d+)\s\((?<percent>\d+)%\)/,
dash_vmaf:
~r/^-\scrf\s(?<crf>\d+(?:\.\d+)?)\sVMAF\s(?<score>\d+\.\d+)\s\((?<percent>\d+)%\)/,
eta_vmaf:
~r/crf\s(?<crf>\d+(?:\.\d+)?)\sVMAF\s(?<score>\d+\.\d+)\spredicted\svideo\sstream\ssize\s(?<size>\d+\.?\d*)\s(?<unit>\w+)\s\((?<percent>\d+)%\)\staking\s(?<time>\d+\.?\d*)\s(?<time_unit>\w+)/,
vmaf_comparison: ~r/vmaf\s(?<file1>.+?)\svs\sreference\s(?<file2>.+)/,
progress:
~r/\[(?<timestamp>[^\]]+)\].*?(?<progress>\d+(?:\.\d+)?)%,\s(?<fps>\d+(?:\.\d+)?)\sfps?,\seta\s(?<eta>\d+)\s(?<time_unit>second|minute|hour|day|week|month|year)s?/,
success: ~r/(?:\[.*\]\s)?crf\s(?<crf>\d+(?:\.\d+)?)\ssuccessful/,
warning: ~r/^Warning:\s(?<message>.*)/,
encoding_start: ~r/\[.*\] encoding (?<filename>\d+\.mkv)/,
encoding_progress:
~r/\[.*\]\s*(?<percent>\d+)%,\s*(?<fps>[\d\.]+)\s*fps,\s*eta\s*(?<eta>\d+)\s*(?<unit>minutes|seconds|hours|days|weeks|months|years)/,
encoding_progress_alt:
~r/(?<percent>\d+)%,\s*(?<fps>[\d\.]+)\s*fps,\s*eta\s*(?<eta>\d+)\s*(?<unit>minutes|seconds|hours|days|weeks|months|years)/,
file_size_progress: ~r/Encoded\s(?<size>[\d\.]+\s\w+)\s\((?<percent>\d+)%\)/,
ffmpeg_error: ~r/Error: ffmpeg encode exit code (?<exit_code>\d+)/
}
end

# Cache patterns in process dictionary for performance
# This avoids recompilation while maintaining Elixir 1.19 compatibility
defp cached_patterns do
case Process.get(:ab_av1_patterns) do
nil ->
patterns = get_patterns()
Process.put(:ab_av1_patterns, patterns)
patterns

patterns ->
patterns
end
end

@doc """
Matches a line against a specific pattern and returns named captures.
Expand All @@ -48,7 +66,8 @@ defmodule Reencodarr.AbAv1.OutputParser do
"""
@spec match_pattern(String.t(), atom()) :: {:ok, map()} | {:error, :no_match}
def match_pattern(line, pattern_key) do
pattern = Map.get(@patterns, pattern_key)
patterns = cached_patterns()
pattern = Map.get(patterns, pattern_key)

case Regex.named_captures(pattern, line) do
nil -> {:error, :no_match}
Expand Down Expand Up @@ -111,15 +130,18 @@ defmodule Reencodarr.AbAv1.OutputParser do
end

defp parse_pattern_with_mapping(line, pattern_key, type, field_mapping) do
case Parsers.parse_with_pattern(line, pattern_key, @patterns, field_mapping) do
patterns = cached_patterns()

case Parsers.parse_with_pattern(line, pattern_key, patterns, field_mapping) do
{:error, _} = error -> error
{:ok, data} -> {:ok, %{type: type, data: data}}
end
end

# Special parser for encoding_start pattern with custom transformations
defp parse_encoding_start_pattern(line) do
pattern = @patterns[:encoding_start]
patterns = cached_patterns()
pattern = patterns[:encoding_start]

case Regex.named_captures(pattern, line) do
nil ->
Expand Down
Loading
Loading