diff --git a/Gemfile b/Gemfile index bf238120ff..2d46350bda 100644 --- a/Gemfile +++ b/Gemfile @@ -30,8 +30,7 @@ gem 'terser' gem 'shakapacker' # Core Samvera -# gem 'active-fedora', '~> 15.0' -gem 'active-fedora', git: 'https://github.com/samvera/active_fedora.git', ref: '7f91e09e630f7e3c1eb3d355e5a016ae8af44778' +gem 'active-fedora', '~> 16.0' gem 'active_fedora-datastreams', '~> 0.5' gem 'hydra-head', '~> 13.0' gem 'ldp', '~> 1.1.0' @@ -73,8 +72,7 @@ gem 'omniauth-lti', git: "https://github.com/avalonmediasystem/omniauth-lti.git" gem "omniauth-saml", "~> 2.0", ">= 2.2.3" # Media Access & Transcoding -#gem 'active_encode', '~> 1.3.0' -gem 'active_encode', git: "https://github.com/samvera-labs/active_encode.git", branch: "mediaconvert_file" +gem 'active_encode', '~> 2.0' gem 'audio_waveform-ruby', '~> 1.0.7', require: 'audio_waveform' gem 'browse-everything', git: "https://github.com/avalonmediasystem/browse-everything.git", tag: 'v1.5-Avalon' gem 'fastimage' diff --git a/Gemfile.lock b/Gemfile.lock index f0e51a855f..5a9d91b987 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -59,24 +59,6 @@ GIT addressable (~> 2.8) rails -GIT - remote: https://github.com/samvera/active_fedora.git - revision: 7f91e09e630f7e3c1eb3d355e5a016ae8af44778 - ref: 7f91e09e630f7e3c1eb3d355e5a016ae8af44778 - specs: - active-fedora (15.0.1) - active-triples (>= 0.11.0, < 2.0.0) - activemodel (>= 6.1) - activesupport (>= 6.1) - deprecation - faraday (>= 2.0) - faraday-encoding (>= 0.0.5) - faraday-follow_redirects - ldp (>= 0.7.0, < 2) - mutex_m - rsolr (>= 1.1.2, < 3) - ruby-progressbar (~> 1.0) - GEM remote: https://rubygems.org/ specs: @@ -123,6 +105,18 @@ GEM erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) + active-fedora (16.0.0) + active-triples (>= 0.11.0, < 2.0.0) + activemodel (>= 6.1) + activesupport (>= 6.1) + deprecation + faraday (>= 2.0) + faraday-encoding (>= 0.0.5) + faraday-follow_redirects + ldp (>= 0.7.0, < 2) + mutex_m + rsolr (>= 1.1.2, < 3) + ruby-progressbar (~> 1.0) active-triples (1.2.0) activemodel (>= 3.0.0) activesupport (>= 3.0.0) @@ -132,6 +126,10 @@ GEM json-ld rails (>= 5.2) rdf-vocab (>= 2.1.0) + active_encode (2.0.0) + addressable (~> 2.8) + rails + retriable active_fedora-datastreams (0.5.0) active-fedora (>= 11.0.0.pre) activemodel (>= 5.2) @@ -252,7 +250,7 @@ GEM bcp47_spec (0.2.1) bcrypt (3.1.20) benchmark (0.4.1) - bigdecimal (3.2.2) + bigdecimal (3.2.3) bindex (0.8.1) bixby (5.0.2) rubocop (= 1.28.2) @@ -329,7 +327,7 @@ GEM config (5.5.2) deep_merge (~> 1.2, >= 1.2.1) ostruct - connection_pool (2.5.3) + connection_pool (2.5.4) crack (1.0.0) bigdecimal rexml @@ -576,7 +574,7 @@ GEM mysql2 (0.5.6) net-http (0.6.0) uri - net-imap (0.5.9) + net-imap (0.5.10) date net-protocol net-ldap (0.19.0) @@ -1000,9 +998,9 @@ PLATFORMS DEPENDENCIES about_page! - active-fedora! + active-fedora (~> 16.0) active_annotations (~> 0.6) - active_encode! + active_encode (~> 2.0) active_fedora-datastreams (~> 0.5) activejob-traffic_control activejob-uniqueness diff --git a/app/models/derivative.rb b/app/models/derivative.rb index fd5ee6fcf1..bf4526c789 100644 --- a/app/models/derivative.rb +++ b/app/models/derivative.rb @@ -78,7 +78,8 @@ def set_streaming_locations! if managed path = Addressable::URI.parse(absolute_location).path self.location_url = Avalon::StreamMapper.stream_path(path) - self.hls_url = Avalon::StreamMapper.map(path, 'http', format) + is_mp3 = format == "audio" && audio_codec == "mp3" + self.hls_url = Avalon::StreamMapper.map(path, 'http', (is_mp3 ? "audio_mp3" : format)) end self end @@ -116,6 +117,10 @@ def self.from_output(output, managed = true) derivative.video_codec = output[:video_codec] derivative.resolution = "#{output[:width]}x#{output[:height]}" if output[:width] && output[:height] + if derivative.format == "audio" && derivative.audio_codec == "mp3" + derivative.mime_type ||= "audio/mpeg" + end + # FIXME: Transform to stream url here? How do we distribute to the streaming server? derivative.location_url = output[:url] # For Intercom push diff --git a/app/services/file_locator.rb b/app/services/file_locator.rb index 816e71f3cd..0bb719bd75 100644 --- a/app/services/file_locator.rb +++ b/app/services/file_locator.rb @@ -134,12 +134,25 @@ def reader when 's3' S3File.new(uri).object.get.body when 'file' - File.open(location,'r') + File.open(location, 'r') else open_uri end end + def magic_bytes + # Magic bytes for relevant file types should max out at 16, so only request + # that range when checking mimetype. + case uri.scheme + when 's3' + S3File.new(uri).object.get(range: 'bytes=0-15').body + when 'file' + File.read(location, 16) + else + open_uri.read(16) + end + end + # Ruby 3.0 removed URI#open from being called by Kernel#open. # Prioritize using URI#open, attempt to fallback to Kernel#open # if URI fails. diff --git a/config/media_convert_presets/avalon_audio_high.json b/config/media_convert_presets/avalon_audio_high.json new file mode 100644 index 0000000000..dc462fd88a --- /dev/null +++ b/config/media_convert_presets/avalon_audio_high.json @@ -0,0 +1,21 @@ +{ + "Name": "avalon_audio_high", + "Category": "avalon", + "Settings": { + "AudioDescriptions": [ + { + "CodecSettings": { + "AacSettings": { + "Bitrate": 320000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + }, + "Codec": "AAC" + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/media_convert_presets/avalon_audio_medium.json b/config/media_convert_presets/avalon_audio_medium.json new file mode 100644 index 0000000000..476df4c788 --- /dev/null +++ b/config/media_convert_presets/avalon_audio_medium.json @@ -0,0 +1,21 @@ +{ + "Name": "avalon_audio_medium", + "Category": "avalon", + "Settings": { + "AudioDescriptions": [ + { + "CodecSettings": { + "AacSettings": { + "Bitrate": 128000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + }, + "Codec": "AAC" + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/media_convert_presets/avalon_video_high.json b/config/media_convert_presets/avalon_video_high.json new file mode 100644 index 0000000000..e6a4fc4fe2 --- /dev/null +++ b/config/media_convert_presets/avalon_video_high.json @@ -0,0 +1,34 @@ +{ + "Name": "avalon_video_high", + "Category": "avalon", + "Settings": { + "VideoDescription": { + "Width": 1920, + "ScalingBehavior": "FIT_NO_UPSCALE", + "Height": 1080, + "CodecSettings": { + "Codec": "H_264", + "H264Settings": { + "Bitrate": 2048000, + "CodecProfile": "HIGH", + "CodecLevel": "AUTO" + } + } + }, + "AudioDescriptions": [ + { + "CodecSettings": { + "Codec": "AAC", + "AacSettings": { + "Bitrate": 192000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + } + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/media_convert_presets/avalon_video_low.json b/config/media_convert_presets/avalon_video_low.json new file mode 100644 index 0000000000..1ad71c225e --- /dev/null +++ b/config/media_convert_presets/avalon_video_low.json @@ -0,0 +1,34 @@ +{ + "Name": "avalon_video_low", + "Category": "avalon", + "Settings": { + "VideoDescription": { + "Width": 720, + "ScalingBehavior": "FIT_NO_UPSCALE", + "Height": 480, + "CodecSettings": { + "Codec": "H_264", + "H264Settings": { + "Bitrate": 500000, + "CodecProfile": "HIGH", + "CodecLevel": "AUTO" + } + } + }, + "AudioDescriptions": [ + { + "CodecSettings": { + "Codec": "AAC", + "AacSettings": { + "Bitrate": 128000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + } + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/media_convert_presets/avalon_video_medium.json b/config/media_convert_presets/avalon_video_medium.json new file mode 100644 index 0000000000..5e94fc9878 --- /dev/null +++ b/config/media_convert_presets/avalon_video_medium.json @@ -0,0 +1,34 @@ +{ + "Name": "avalon_video_medium", + "Category": "avalon", + "Settings": { + "VideoDescription": { + "Width": 1280, + "ScalingBehavior": "FIT_NO_UPSCALE", + "Height": 720, + "CodecSettings": { + "Codec": "H_264", + "H264Settings": { + "Bitrate": 1024000, + "CodecProfile": "HIGH", + "CodecLevel": "AUTO" + } + } + }, + "AudioDescriptions": [ + { + "CodecSettings": { + "Codec": "AAC", + "AacSettings": { + "Bitrate": 128000, + "CodingMode": "CODING_MODE_2_0", + "SampleRate": 44100 + } + } + } + ], + "ContainerSettings": { + "Container": "MP4" + } + } +} diff --git a/config/url_handlers.yml b/config/url_handlers.yml index 6391299641..01e156ee68 100644 --- a/config/url_handlers.yml +++ b/config/url_handlers.yml @@ -15,4 +15,5 @@ nginx: http: video: <%=http_base%>/<%=path%>/<%=filename%>.<%=extension%>/index.m3u8 audio: <%=http_base%>/<%=path%>/<%=filename%>.<%=extension%>/index.m3u8 + audio_mp3: <%=http_base%>/<%=path%>/<%=filename%>.<%=extension%> diff --git a/docker-compose.yml b/docker-compose.yml index 92ebd1f46b..9534984587 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,10 +65,11 @@ services: - ./solr/conf:/opt/solr/avalon_conf hls: - image: avalonmediasystem/nginx:minio-jammy + image: avalonmediasystem/nginx:noble environment: - AVALON_DOMAIN=http://avalon:3000 - AVALON_STREAMING_BUCKET_URL=http://minio:9000/derivatives/ + - VOD_MODE=remote volumes: - ./log/nginx:/var/log/nginx ports: diff --git a/lib/avalon/ffprobe.rb b/lib/avalon/ffprobe.rb index 7f2ea8bc19..b660342604 100644 --- a/lib/avalon/ffprobe.rb +++ b/lib/avalon/ffprobe.rb @@ -80,7 +80,7 @@ def content_type media_file # Remove S3 credentials or other params from extension output extension = File.extname(media_file.location)&.gsub(/[\?#].*/, '') # Fall back on file extension if magic bytes fail to identify file - Marcel::MimeType.for media_file.reader, extension: extension + @content_type ||= Marcel::MimeType.for media_file.magic_bytes, extension: extension end def valid_content_type? media_file diff --git a/lib/avalon/media_convert_encode.rb b/lib/avalon/media_convert_encode.rb index 38639472db..d18945c259 100644 --- a/lib/avalon/media_convert_encode.rb +++ b/lib/avalon/media_convert_encode.rb @@ -12,8 +12,6 @@ # specific language governing permissions and limitations under the License. # --- END LICENSE_HEADER BLOCK --- -require 'avalon/elastic_transcoder' - class MediaConvertEncode < WatchedEncode self.engine_adapter = :media_convert self.engine_adapter.role = Settings.encoding.media_convert_role diff --git a/spec/services/file_locator_spec.rb b/spec/services/file_locator_spec.rb index bf4cd5430c..398263394e 100644 --- a/spec/services/file_locator_spec.rb +++ b/spec/services/file_locator_spec.rb @@ -45,7 +45,7 @@ allow(ENV).to receive(:[]).with('AWS_REGION').and_return('us-east-2') expect(subject.host).to eq 'mybucket.domain' expect(subject.path).to eq '/mykey.mp4' - expect(subject.query).to include('response-content-disposition=attachment%3B%20filename%3Dmykey.mp4', 'X-Amz-Algorithm', + expect(subject.query).to include('response-content-disposition=attachment%3B%20filename%3Dmykey.mp4', 'X-Amz-Algorithm', 'X-Amz-Credential', 'X-Amz-Expires', 'X-Amz-SignedHeaders', 'X-Amz-Signature') end @@ -60,7 +60,7 @@ allow(ENV).to receive(:[]).with('AWS_REGION').and_return('us-east-2') expect(subject.host).to eq 'mybucket.s3.us-stubbed-1.amazonaws.com' expect(subject.path).to eq '/mykey.mp4' - expect(subject.query).to include('response-content-disposition=attachment%3B%20filename%3Dmykey.mp4', 'X-Amz-Algorithm', + expect(subject.query).to include('response-content-disposition=attachment%3B%20filename%3Dmykey.mp4', 'X-Amz-Algorithm', 'X-Amz-Credential', 'X-Amz-Expires', 'X-Amz-SignedHeaders', 'X-Amz-Signature') end end @@ -168,7 +168,7 @@ describe '#remove_fs_dir' do let(:file_path) { "/tmp/dropbox/test/mykey.mp4" } let(:locator) { FileLocator.new(file_path) } - + it 'deletes fs dir' do allow(File).to receive(:exist?).with(file_path) { true } expect(locator.exist?).to be_truthy @@ -177,14 +177,14 @@ expect(locator.exist?).to be_falsey end end - + describe '#remove_s3_dir' do let(:old_bucket) { Settings.encoding.masterfile_bucket } let(:old_path) { Settings.dropbox.path } let(:dropbox_path) { "s3://#{test_bucket}/dropbox/test_collection" } let(:test_bucket) { "test_bucket" } - let(:dropbox_prefix) { "/dropbox/test_collection/"} + let(:dropbox_prefix) { "/dropbox/test_collection/" } let(:s3_res) { Aws::S3::Resource.new } let(:s3_bucket) { Aws::S3::Bucket.new(test_bucket) } @@ -225,4 +225,58 @@ end end end + + describe "#magic_bytes" do + let(:locator) { FileLocator.new(source) } + let(:source) { "file://#{Rails.root.join('spec', 'fixtures', 'videoshort.mp4')}" } + + context "local file" do + it "returns the first 16 bytes" do + expect(locator.magic_bytes.length).to eq 16 + end + + it 'successfully returns correct mimetype' do + expect(Marcel::MimeType.for(locator.magic_bytes)).to eq 'video/mp4' + end + end + + context "s3 file" do + let(:client) { Aws::S3::Client.new(stub_responses: true) } + let(:bucket) { "mybucket" } + let(:key) { "meow.wav" } + let(:source) { "s3://#{bucket}/#{key}" } + let(:object_body) { File.read(Rails.root.join('spec', 'fixtures', 'videoshort.mp4')) } + + before do + client.stub_responses(:get_object, lambda { |context| + range = context.params[:range] + from, to = range.match(/bytes=(\d+)-(\d+)/)[1..2].map(&:to_i) + # Return the specific byte range of the content + { body: object_body[from..to], content_range: "bytes #{from}-#{to}/#{object_body.length}" } + }) + + allow(Aws::S3::Client).to receive(:new).and_return(client) + end + + it "returns the first 16 bytes" do + expect(locator.magic_bytes.length).to eq 16 + end + + it 'successfully returns correct mimetype' do + expect(Marcel::MimeType.for(locator.magic_bytes)).to eq 'video/mp4' + end + end + + context "other file" do + let(:file_path) { "bogus://#{source}" } + it "returns the first 16 bytes" do + allow(URI).to receive(:open).with(file_path).and_return(URI.open(Rails.root.join('spec', 'fixtures', 'videoshort.mp4').to_s)) + expect(locator.magic_bytes.length).to eq 16 + end + + it 'successfully returns correct mimetype' do + expect(Marcel::MimeType.for(locator.magic_bytes)).to eq 'video/mp4' + end + end + end end