From a65cff78a60096e3181778150b4919a14c07694f Mon Sep 17 00:00:00 2001 From: cjcolvar Date: Mon, 8 Dec 2025 09:27:09 -0500 Subject: [PATCH 1/7] Bump to released ActiveFedora with Fedora 6 support --- Gemfile | 3 +-- Gemfile.lock | 32 +++++++++++++------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/Gemfile b/Gemfile index bf238120ff..63e98e03e1 100644 --- a/Gemfile +++ b/Gemfile @@ -30,8 +30,7 @@ gem 'terser' gem 'shakapacker' # Core Samvera -# gem 'active-fedora', '~> 15.0' -gem 'active-fedora', git: 'https://github.com/samvera/active_fedora.git', ref: '7f91e09e630f7e3c1eb3d355e5a016ae8af44778' +gem 'active-fedora', '~> 16.0' gem 'active_fedora-datastreams', '~> 0.5' gem 'hydra-head', '~> 13.0' gem 'ldp', '~> 1.1.0' diff --git a/Gemfile.lock b/Gemfile.lock index f0e51a855f..64a0cf8f8f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -59,24 +59,6 @@ GIT addressable (~> 2.8) rails -GIT - remote: https://github.com/samvera/active_fedora.git - revision: 7f91e09e630f7e3c1eb3d355e5a016ae8af44778 - ref: 7f91e09e630f7e3c1eb3d355e5a016ae8af44778 - specs: - active-fedora (15.0.1) - active-triples (>= 0.11.0, < 2.0.0) - activemodel (>= 6.1) - activesupport (>= 6.1) - deprecation - faraday (>= 2.0) - faraday-encoding (>= 0.0.5) - faraday-follow_redirects - ldp (>= 0.7.0, < 2) - mutex_m - rsolr (>= 1.1.2, < 3) - ruby-progressbar (~> 1.0) - GEM remote: https://rubygems.org/ specs: @@ -123,6 +105,18 @@ GEM erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) + active-fedora (16.0.0) + active-triples (>= 0.11.0, < 2.0.0) + activemodel (>= 6.1) + activesupport (>= 6.1) + deprecation + faraday (>= 2.0) + faraday-encoding (>= 0.0.5) + faraday-follow_redirects + ldp (>= 0.7.0, < 2) + mutex_m + rsolr (>= 1.1.2, < 3) + ruby-progressbar (~> 1.0) active-triples (1.2.0) activemodel (>= 3.0.0) activesupport (>= 3.0.0) @@ -1000,7 +994,7 @@ PLATFORMS DEPENDENCIES about_page! - active-fedora! + active-fedora (~> 16.0) active_annotations (~> 0.6) active_encode! active_fedora-datastreams (~> 0.5) From bb2e5f0fc0f1bd9a6a87128f261f7cebd77c1bfe Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Tue, 9 Dec 2025 17:13:34 -0500 Subject: [PATCH 2/7] Only request magic bytes for mimetype verification Related issue: #6649 We analyze input files on upload to determine if they are audio or video and to grab some of the metadata to fill out the record. This process involved checking the file mimetype, which was using our `FileLocator.reader` method. In S3 environments this would download the entire file, resulting in linearly increasing upload times based on file size. By requesting just the first ~16 bytes of the file for this case, we should be able to reduce upload time of multi-gig files from several minutes down to less than 10 seconds. --- app/services/file_locator.rb | 13 ++++++++++++- lib/avalon/ffprobe.rb | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/app/services/file_locator.rb b/app/services/file_locator.rb index 816e71f3cd..5af2787bfe 100644 --- a/app/services/file_locator.rb +++ b/app/services/file_locator.rb @@ -134,7 +134,18 @@ def reader when 's3' S3File.new(uri).object.get.body when 'file' - File.open(location,'r') + File.open(location, 'r') + else + open_uri + end + end + + def magic_bytes + case uri.scheme + when 's3' + S3File.new(uri).object.get(range: '0-16').body + when 'file' + File.open(location, 'r') else open_uri end diff --git a/lib/avalon/ffprobe.rb b/lib/avalon/ffprobe.rb index 7f2ea8bc19..b660342604 100644 --- a/lib/avalon/ffprobe.rb +++ b/lib/avalon/ffprobe.rb @@ -80,7 +80,7 @@ def content_type media_file # Remove S3 credentials or other params from extension output extension = File.extname(media_file.location)&.gsub(/[\?#].*/, '') # Fall back on file extension if magic bytes fail to identify file - Marcel::MimeType.for media_file.reader, extension: extension + @content_type ||= Marcel::MimeType.for media_file.magic_bytes, extension: extension end def valid_content_type? media_file From b6fda503bae76e3f25f845c1a71445a97d882a5a Mon Sep 17 00:00:00 2001 From: Mason Ballengee Date: Thu, 11 Dec 2025 15:08:53 -0500 Subject: [PATCH 3/7] Limit magic_bytes response size for all file types, add tests Co-authored-by: Chris Colvard --- app/services/file_locator.rb | 8 ++-- spec/services/file_locator_spec.rb | 64 +++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/app/services/file_locator.rb b/app/services/file_locator.rb index 5af2787bfe..0bb719bd75 100644 --- a/app/services/file_locator.rb +++ b/app/services/file_locator.rb @@ -141,13 +141,15 @@ def reader end def magic_bytes + # Magic bytes for relevant file types should max out at 16, so only request + # that range when checking mimetype. case uri.scheme when 's3' - S3File.new(uri).object.get(range: '0-16').body + S3File.new(uri).object.get(range: 'bytes=0-15').body when 'file' - File.open(location, 'r') + File.read(location, 16) else - open_uri + open_uri.read(16) end end diff --git a/spec/services/file_locator_spec.rb b/spec/services/file_locator_spec.rb index bf4cd5430c..398263394e 100644 --- a/spec/services/file_locator_spec.rb +++ b/spec/services/file_locator_spec.rb @@ -45,7 +45,7 @@ allow(ENV).to receive(:[]).with('AWS_REGION').and_return('us-east-2') expect(subject.host).to eq 'mybucket.domain' expect(subject.path).to eq '/mykey.mp4' - expect(subject.query).to include('response-content-disposition=attachment%3B%20filename%3Dmykey.mp4', 'X-Amz-Algorithm', + expect(subject.query).to include('response-content-disposition=attachment%3B%20filename%3Dmykey.mp4', 'X-Amz-Algorithm', 'X-Amz-Credential', 'X-Amz-Expires', 'X-Amz-SignedHeaders', 'X-Amz-Signature') end @@ -60,7 +60,7 @@ allow(ENV).to receive(:[]).with('AWS_REGION').and_return('us-east-2') expect(subject.host).to eq 'mybucket.s3.us-stubbed-1.amazonaws.com' expect(subject.path).to eq '/mykey.mp4' - expect(subject.query).to include('response-content-disposition=attachment%3B%20filename%3Dmykey.mp4', 'X-Amz-Algorithm', + expect(subject.query).to include('response-content-disposition=attachment%3B%20filename%3Dmykey.mp4', 'X-Amz-Algorithm', 'X-Amz-Credential', 'X-Amz-Expires', 'X-Amz-SignedHeaders', 'X-Amz-Signature') end end @@ -168,7 +168,7 @@ describe '#remove_fs_dir' do let(:file_path) { "/tmp/dropbox/test/mykey.mp4" } let(:locator) { FileLocator.new(file_path) } - + it 'deletes fs dir' do allow(File).to receive(:exist?).with(file_path) { true } expect(locator.exist?).to be_truthy @@ -177,14 +177,14 @@ expect(locator.exist?).to be_falsey end end - + describe '#remove_s3_dir' do let(:old_bucket) { Settings.encoding.masterfile_bucket } let(:old_path) { Settings.dropbox.path } let(:dropbox_path) { "s3://#{test_bucket}/dropbox/test_collection" } let(:test_bucket) { "test_bucket" } - let(:dropbox_prefix) { "/dropbox/test_collection/"} + let(:dropbox_prefix) { "/dropbox/test_collection/" } let(:s3_res) { Aws::S3::Resource.new } let(:s3_bucket) { Aws::S3::Bucket.new(test_bucket) } @@ -225,4 +225,58 @@ end end end + + describe "#magic_bytes" do + let(:locator) { FileLocator.new(source) } + let(:source) { "file://#{Rails.root.join('spec', 'fixtures', 'videoshort.mp4')}" } + + context "local file" do + it "returns the first 16 bytes" do + expect(locator.magic_bytes.length).to eq 16 + end + + it 'successfully returns correct mimetype' do + expect(Marcel::MimeType.for(locator.magic_bytes)).to eq 'video/mp4' + end + end + + context "s3 file" do + let(:client) { Aws::S3::Client.new(stub_responses: true) } + let(:bucket) { "mybucket" } + let(:key) { "meow.wav" } + let(:source) { "s3://#{bucket}/#{key}" } + let(:object_body) { File.read(Rails.root.join('spec', 'fixtures', 'videoshort.mp4')) } + + before do + client.stub_responses(:get_object, lambda { |context| + range = context.params[:range] + from, to = range.match(/bytes=(\d+)-(\d+)/)[1..2].map(&:to_i) + # Return the specific byte range of the content + { body: object_body[from..to], content_range: "bytes #{from}-#{to}/#{object_body.length}" } + }) + + allow(Aws::S3::Client).to receive(:new).and_return(client) + end + + it "returns the first 16 bytes" do + expect(locator.magic_bytes.length).to eq 16 + end + + it 'successfully returns correct mimetype' do + expect(Marcel::MimeType.for(locator.magic_bytes)).to eq 'video/mp4' + end + end + + context "other file" do + let(:file_path) { "bogus://#{source}" } + it "returns the first 16 bytes" do + allow(URI).to receive(:open).with(file_path).and_return(URI.open(Rails.root.join('spec', 'fixtures', 'videoshort.mp4').to_s)) + expect(locator.magic_bytes.length).to eq 16 + end + + it 'successfully returns correct mimetype' do + expect(Marcel::MimeType.for(locator.magic_bytes)).to eq 'video/mp4' + end + end + end end From 7145f1391e48f57de4421df5f5e0ad18fd753c88 Mon Sep 17 00:00:00 2001 From: cjcolvar Date: Thu, 4 Sep 2025 17:28:25 -0400 Subject: [PATCH 4/7] Extra handling needed to support streaming mp3 files ingested through skip transcoding. Resolves #6315 --- Gemfile.lock | 6 +++--- app/models/derivative.rb | 7 ++++++- config/url_handlers.yml | 1 + docker-compose.yml | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 64a0cf8f8f..1198800876 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -246,7 +246,7 @@ GEM bcp47_spec (0.2.1) bcrypt (3.1.20) benchmark (0.4.1) - bigdecimal (3.2.2) + bigdecimal (3.2.3) bindex (0.8.1) bixby (5.0.2) rubocop (= 1.28.2) @@ -323,7 +323,7 @@ GEM config (5.5.2) deep_merge (~> 1.2, >= 1.2.1) ostruct - connection_pool (2.5.3) + connection_pool (2.5.4) crack (1.0.0) bigdecimal rexml @@ -570,7 +570,7 @@ GEM mysql2 (0.5.6) net-http (0.6.0) uri - net-imap (0.5.9) + net-imap (0.5.10) date net-protocol net-ldap (0.19.0) diff --git a/app/models/derivative.rb b/app/models/derivative.rb index fd5ee6fcf1..bf4526c789 100644 --- a/app/models/derivative.rb +++ b/app/models/derivative.rb @@ -78,7 +78,8 @@ def set_streaming_locations! if managed path = Addressable::URI.parse(absolute_location).path self.location_url = Avalon::StreamMapper.stream_path(path) - self.hls_url = Avalon::StreamMapper.map(path, 'http', format) + is_mp3 = format == "audio" && audio_codec == "mp3" + self.hls_url = Avalon::StreamMapper.map(path, 'http', (is_mp3 ? "audio_mp3" : format)) end self end @@ -116,6 +117,10 @@ def self.from_output(output, managed = true) derivative.video_codec = output[:video_codec] derivative.resolution = "#{output[:width]}x#{output[:height]}" if output[:width] && output[:height] + if derivative.format == "audio" && derivative.audio_codec == "mp3" + derivative.mime_type ||= "audio/mpeg" + end + # FIXME: Transform to stream url here? How do we distribute to the streaming server? derivative.location_url = output[:url] # For Intercom push diff --git a/config/url_handlers.yml b/config/url_handlers.yml index 6391299641..01e156ee68 100644 --- a/config/url_handlers.yml +++ b/config/url_handlers.yml @@ -15,4 +15,5 @@ nginx: http: video: <%=http_base%>/<%=path%>/<%=filename%>.<%=extension%>/index.m3u8 audio: <%=http_base%>/<%=path%>/<%=filename%>.<%=extension%>/index.m3u8 + audio_mp3: <%=http_base%>/<%=path%>/<%=filename%>.<%=extension%> diff --git a/docker-compose.yml b/docker-compose.yml index 92ebd1f46b..9534984587 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,10 +65,11 @@ services: - ./solr/conf:/opt/solr/avalon_conf hls: - image: avalonmediasystem/nginx:minio-jammy + image: avalonmediasystem/nginx:noble environment: - AVALON_DOMAIN=http://avalon:3000 - AVALON_STREAMING_BUCKET_URL=http://minio:9000/derivatives/ + - VOD_MODE=remote volumes: - ./log/nginx:/var/log/nginx ports: From 63d143b05134b7b98328c4d90a269737cff996cb Mon Sep 17 00:00:00 2001 From: cjcolvar Date: Wed, 17 Sep 2025 11:34:10 -0400 Subject: [PATCH 5/7] Add support for AWS Elemental MediaConvert active_encode adapter --- Gemfile | 3 +-- Gemfile.lock | 6 +++++- lib/avalon/media_convert_encode.rb | 2 -- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Gemfile b/Gemfile index 63e98e03e1..2d46350bda 100644 --- a/Gemfile +++ b/Gemfile @@ -72,8 +72,7 @@ gem 'omniauth-lti', git: "https://github.com/avalonmediasystem/omniauth-lti.git" gem "omniauth-saml", "~> 2.0", ">= 2.2.3" # Media Access & Transcoding -#gem 'active_encode', '~> 1.3.0' -gem 'active_encode', git: "https://github.com/samvera-labs/active_encode.git", branch: "mediaconvert_file" +gem 'active_encode', '~> 2.0' gem 'audio_waveform-ruby', '~> 1.0.7', require: 'audio_waveform' gem 'browse-everything', git: "https://github.com/avalonmediasystem/browse-everything.git", tag: 'v1.5-Avalon' gem 'fastimage' diff --git a/Gemfile.lock b/Gemfile.lock index 1198800876..5a9d91b987 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -126,6 +126,10 @@ GEM json-ld rails (>= 5.2) rdf-vocab (>= 2.1.0) + active_encode (2.0.0) + addressable (~> 2.8) + rails + retriable active_fedora-datastreams (0.5.0) active-fedora (>= 11.0.0.pre) activemodel (>= 5.2) @@ -996,7 +1000,7 @@ DEPENDENCIES about_page! active-fedora (~> 16.0) active_annotations (~> 0.6) - active_encode! + active_encode (~> 2.0) active_fedora-datastreams (~> 0.5) activejob-traffic_control activejob-uniqueness diff --git a/lib/avalon/media_convert_encode.rb b/lib/avalon/media_convert_encode.rb index 38639472db..d18945c259 100644 --- a/lib/avalon/media_convert_encode.rb +++ b/lib/avalon/media_convert_encode.rb @@ -12,8 +12,6 @@ # specific language governing permissions and limitations under the License. # --- END LICENSE_HEADER BLOCK --- -require 'avalon/elastic_transcoder' - class MediaConvertEncode < WatchedEncode self.engine_adapter = :media_convert self.engine_adapter.role = Settings.encoding.media_convert_role From ad6330cf6c206cb08f48e0cfcfe83ec9f6b696ff Mon Sep 17 00:00:00 2001 From: cjcolvar Date: Thu, 20 Nov 2025 16:04:46 -0500 Subject: [PATCH 6/7] Add missing media convert presets --- .../avalon_audio_high.json | 21 ++++++++++++ .../avalon_audio_medium.json | 21 ++++++++++++ .../avalon_video_high.json | 34 +++++++++++++++++++ .../avalon_video_low.json | 34 +++++++++++++++++++ .../avalon_video_medium.json | 34 +++++++++++++++++++ 5 files changed, 144 insertions(+) create mode 100644 config/media_convert_presets/avalon_audio_high.json create mode 100644 config/media_convert_presets/avalon_audio_medium.json create mode 100644 config/media_convert_presets/avalon_video_high.json create mode 100644 config/media_convert_presets/avalon_video_low.json create mode 100644 config/media_convert_presets/avalon_video_medium.json diff --git a/config/media_convert_presets/avalon_audio_high.json b/config/media_convert_presets/avalon_audio_high.json new file mode 100644 index 0000000000..dc462fd88a --- /dev/null +++ b/config/media_convert_presets/avalon_audio_high.json @@ -0,0 +1,21 @@ +{ + "Name": "avalon_audio_high", + "Category": "avalon", + "Settings": { + "AudioDescriptions": [ + { + "CodecSettings": { + "AacSettings": { + "Bitrate": 320000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + }, + "Codec": "AAC" + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/media_convert_presets/avalon_audio_medium.json b/config/media_convert_presets/avalon_audio_medium.json new file mode 100644 index 0000000000..476df4c788 --- /dev/null +++ b/config/media_convert_presets/avalon_audio_medium.json @@ -0,0 +1,21 @@ +{ + "Name": "avalon_audio_medium", + "Category": "avalon", + "Settings": { + "AudioDescriptions": [ + { + "CodecSettings": { + "AacSettings": { + "Bitrate": 128000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + }, + "Codec": "AAC" + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/media_convert_presets/avalon_video_high.json b/config/media_convert_presets/avalon_video_high.json new file mode 100644 index 0000000000..0dda1e3cf3 --- /dev/null +++ b/config/media_convert_presets/avalon_video_high.json @@ -0,0 +1,34 @@ +{ + "Name": "avalon_video_high", + "Category": "avalon", + "Settings": { + "VideoDescription": { + "Width": 1920, + "ScalingBehavior": "FIT_NO_UPSCALE", + "Height": 1080, + "CodecSettings": { + "Codec": "H_264", + "H264Settings": { + "Bitrate": 2048000, + "CodecProfile": "MAIN", + "CodecLevel": "AUTO" + } + } + }, + "AudioDescriptions": [ + { + "CodecSettings": { + "Codec": "AAC", + "AacSettings": { + "Bitrate": 192000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + } + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/media_convert_presets/avalon_video_low.json b/config/media_convert_presets/avalon_video_low.json new file mode 100644 index 0000000000..f7f0dfc9cd --- /dev/null +++ b/config/media_convert_presets/avalon_video_low.json @@ -0,0 +1,34 @@ +{ + "Name": "avalon_video_low", + "Category": "avalon", + "Settings": { + "VideoDescription": { + "Width": 720, + "ScalingBehavior": "FIT_NO_UPSCALE", + "Height": 480, + "CodecSettings": { + "Codec": "H_264", + "H264Settings": { + "Bitrate": 500000, + "CodecProfile": "MAIN", + "CodecLevel": "LEVEL_3_1" + } + } + }, + "AudioDescriptions": [ + { + "CodecSettings": { + "Codec": "AAC", + "AacSettings": { + "Bitrate": 128000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + } + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/media_convert_presets/avalon_video_medium.json b/config/media_convert_presets/avalon_video_medium.json new file mode 100644 index 0000000000..3c1bc089e9 --- /dev/null +++ b/config/media_convert_presets/avalon_video_medium.json @@ -0,0 +1,34 @@ +{ + "Name": "avalon_video_medium", + "Category": "avalon", + "Settings": { + "VideoDescription": { + "Width": 1280, + "ScalingBehavior": "FIT_NO_UPSCALE", + "Height": 720, + "CodecSettings": { + "Codec": "H_264", + "H264Settings": { + "Bitrate": 1024000, + "CodecProfile": "MAIN", + "CodecLevel": "LEVEL_3_1" + } + } + }, + "AudioDescriptions": [ + { + "CodecSettings": { + "Codec": "AAC", + "AacSettings": { + "Bitrate": 128000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + } + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} From 804661c5bff3d17030fc67bd74a81c7943f51348 Mon Sep 17 00:00:00 2001 From: cjcolvar Date: Tue, 13 Jan 2026 11:46:07 -0500 Subject: [PATCH 7/7] Change codec profile to high and codec level to auto for all presets --- config/media_convert_presets/avalon_video_high.json | 2 +- config/media_convert_presets/avalon_video_low.json | 4 ++-- config/media_convert_presets/avalon_video_medium.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/media_convert_presets/avalon_video_high.json b/config/media_convert_presets/avalon_video_high.json index 0dda1e3cf3..e6a4fc4fe2 100644 --- a/config/media_convert_presets/avalon_video_high.json +++ b/config/media_convert_presets/avalon_video_high.json @@ -10,7 +10,7 @@ "Codec": "H_264", "H264Settings": { "Bitrate": 2048000, - "CodecProfile": "MAIN", + "CodecProfile": "HIGH", "CodecLevel": "AUTO" } } diff --git a/config/media_convert_presets/avalon_video_low.json b/config/media_convert_presets/avalon_video_low.json index f7f0dfc9cd..1ad71c225e 100644 --- a/config/media_convert_presets/avalon_video_low.json +++ b/config/media_convert_presets/avalon_video_low.json @@ -10,8 +10,8 @@ "Codec": "H_264", "H264Settings": { "Bitrate": 500000, - "CodecProfile": "MAIN", - "CodecLevel": "LEVEL_3_1" + "CodecProfile": "HIGH", + "CodecLevel": "AUTO" } } }, diff --git a/config/media_convert_presets/avalon_video_medium.json b/config/media_convert_presets/avalon_video_medium.json index 3c1bc089e9..5e94fc9878 100644 --- a/config/media_convert_presets/avalon_video_medium.json +++ b/config/media_convert_presets/avalon_video_medium.json @@ -10,8 +10,8 @@ "Codec": "H_264", "H264Settings": { "Bitrate": 1024000, - "CodecProfile": "MAIN", - "CodecLevel": "LEVEL_3_1" + "CodecProfile": "HIGH", + "CodecLevel": "AUTO" } } },