diff --git a/src/container_repair/mod.rs b/src/container_repair/mod.rs new file mode 100644 index 0000000..e5ef147 --- /dev/null +++ b/src/container_repair/mod.rs @@ -0,0 +1,104 @@ +//! Detection and in-flight repair of malformed audio container headers. +//! +//! Some audio files carry container-level defects (e.g. a truncated MP4 whose +//! `mdat` box size overruns the real end of the file) that make symphonia +//! reject them even though the audio present decodes fine. Each format-specific +//! submodule detects one class of defect and describes the fix as byte +//! [`Patch`]es; [`PatchedSource`] overlays those patches onto the file as it is +//! read, so the file on disk is never modified. +//! +//! To support a new format repair, add a submodule and a match arm in +//! [`detect_repairs`]. + +mod mp4; + +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; +use symphonia::core::io::MediaSource; + +/// A single-byte overlay: when reading, the byte at `offset` is replaced by +/// `value`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Patch { + /// Absolute file offset of the byte to replace. + pub offset: u64, + /// Replacement byte. + pub value: u8, +} + +/// Scan `file` (whose format is chosen by `path`'s extension) for known +/// container defects and return the byte patches that repair them. +/// +/// Returns an empty vec for well-formed files (the common case), so the caller +/// decodes the original file untouched. Only small header regions are read, +/// not the media data, and the file cursor is rewound to the start before +/// returning. +pub fn detect_repairs(file: &File, path: &Path, file_len: u64) -> Vec { + let ext = path + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_ascii_lowercase()); + + let patches = match ext.as_deref() { + Some(e) if mp4::EXTENSIONS.contains(&e) => mp4::mdat_size_patches(file, file_len), + _ => Vec::new(), + }; + + // The scan moves the shared file cursor; rewind so the caller reads from + // the top. If this fails, the caller's own reads will surface the error. + let mut f = file; + let _ = f.seek(SeekFrom::Start(0)); + + patches +} + +/// A [`MediaSource`] that overlays a handful of single-byte [`Patch`]es onto a +/// file as it is read, used to repair a malformed container header (see +/// [`detect_repairs`]). All other bytes pass through unchanged. +pub struct PatchedSource { + inner: File, + len: u64, + patches: Vec, +} + +impl PatchedSource { + /// Wrap `inner` (of total size `len`) so that reads see `patches` applied. + pub fn new(inner: File, len: u64, patches: Vec) -> Self { + Self { + inner, + len, + patches, + } + } +} + +impl Read for PatchedSource { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let start = self.inner.stream_position()?; + let n = self.inner.read(buf)?; + let end = start + n as u64; + for &Patch { offset, value } in &self.patches { + if offset >= start && offset < end { + buf[(offset - start) as usize] = value; + } + } + Ok(n) + } +} + +impl Seek for PatchedSource { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.inner.seek(pos) + } +} + +impl MediaSource for PatchedSource { + fn is_seekable(&self) -> bool { + true + } + + fn byte_len(&self) -> Option { + Some(self.len) + } +} diff --git a/src/container_repair/mp4.rs b/src/container_repair/mp4.rs new file mode 100644 index 0000000..2a0fd16 --- /dev/null +++ b/src/container_repair/mp4.rs @@ -0,0 +1,158 @@ +//! Repairs for ISO-BMFF (MP4/M4A) container defects. + +use super::Patch; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; + +/// MP4/M4A extensions that route to symphonia's isomp4 demuxer and can carry a +/// truncated `mdat`. Raw AAC (`.aac`) is ADTS, not an ISO-BMFF container. +pub(super) const EXTENSIONS: &[&str] = &["m4a", "m4b", "m4p", "mp4", "mov"]; + +/// Scan the top-level atoms of an MP4/M4A file for an `mdat` box whose declared +/// size overruns the end of the file (an interrupted download or copy), and +/// return the byte patches that rewrite its size field to the number of bytes +/// actually present. Symphonia's isomp4 demuxer discards such an mdat wholesale +/// and then fails with "no atom pending read", even though the audio that IS +/// present decodes fine; with the size repaired it hits a clean EOF at the +/// truncation point instead. +/// +/// Returns an empty vec for well-formed files. This only walks the small +/// top-level atom headers, not the media data. The file cursor is left where +/// the scan stopped; the caller is responsible for rewinding. +pub(super) fn mdat_size_patches(file: &File, file_len: u64) -> Vec { + let mut f = file; + let mut pos: u64 = 0; + let mut hdr = [0u8; 16]; + while pos + 8 <= file_len { + if f.seek(SeekFrom::Start(pos)).is_err() || f.read_exact(&mut hdr[..8]).is_err() { + break; + } + let size32 = u32::from_be_bytes([hdr[0], hdr[1], hdr[2], hdr[3]]); + let atom_type = [hdr[4], hdr[5], hdr[6], hdr[7]]; + + // Resolve the atom size and where/how wide its size field is. + let (atom_size, size_off, size_is_64) = if size32 == 1 { + // 64-bit size stored immediately after the type field. + if f.read_exact(&mut hdr[8..16]).is_err() { + break; + } + let size64 = u64::from_be_bytes(hdr[8..16].try_into().unwrap()); + (size64, pos + 8, true) + } else if size32 == 0 { + // Size 0 already means "extends to end of file", and nothing can + // follow it — no repair needed or possible. + break; + } else { + (size32 as u64, pos, false) + }; + + if atom_size < 8 { + break; // malformed header; give up rather than loop + } + + let atom_end = pos.saturating_add(atom_size); + + if atom_type == *b"mdat" && atom_end > file_len { + // Rewrite the size field so the box ends exactly at EOF. + let corrected = file_len - pos; + let bytes_at = |slice: &[u8]| -> Vec { + slice + .iter() + .enumerate() + .map(|(i, &b)| Patch { + offset: size_off + i as u64, + value: b, + }) + .collect() + }; + return if size_is_64 { + bytes_at(&corrected.to_be_bytes()) + } else if corrected <= u32::MAX as u64 { + bytes_at(&(corrected as u32).to_be_bytes()) + } else { + Vec::new() // corrected size won't fit a 32-bit field + }; + } + + pos = atom_end; + } + + Vec::new() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + /// Build a file with an `ftyp` atom followed by an `mdat` atom whose + /// 32-bit size field declares `declared_payload` bytes of media data, + /// while only `actual_payload` bytes are written (truncation). + fn mp4_file(declared_payload: u32, actual_payload: usize) -> tempfile::NamedTempFile { + let mut f = tempfile::NamedTempFile::new().unwrap(); + f.write_all(&16u32.to_be_bytes()).unwrap(); + f.write_all(b"ftypM4A \0\0\0\0").unwrap(); + f.write_all(&(8 + declared_payload).to_be_bytes()).unwrap(); + f.write_all(b"mdat").unwrap(); + f.write_all(&vec![0xAAu8; actual_payload]).unwrap(); + f + } + + fn patches_for(f: &tempfile::NamedTempFile) -> Vec { + let file = f.reopen().unwrap(); + let len = file.metadata().unwrap().len(); + mdat_size_patches(&file, len) + } + + #[test] + fn well_formed_file_needs_no_patches() { + let f = mp4_file(100, 100); + assert!(patches_for(&f).is_empty()); + } + + #[test] + fn truncated_mdat_size_is_rewritten_to_bytes_on_disk() { + // mdat declares 100 payload bytes but only 40 are present. + let f = mp4_file(100, 40); + let patches = patches_for(&f); + + // The 32-bit size field at offset 16 must now read 8 + 40 = 48. + let expected: Vec = 48u32 + .to_be_bytes() + .iter() + .enumerate() + .map(|(i, &b)| Patch { + offset: 16 + i as u64, + value: b, + }) + .collect(); + assert_eq!(patches, expected); + } + + #[test] + fn truncated_64bit_mdat_size_is_rewritten() { + // mdat with size32 == 1 and a 64-bit size that overruns the file. + let mut f = tempfile::NamedTempFile::new().unwrap(); + f.write_all(&16u32.to_be_bytes()).unwrap(); + f.write_all(b"ftypM4A \0\0\0\0").unwrap(); + f.write_all(&1u32.to_be_bytes()).unwrap(); + f.write_all(b"mdat").unwrap(); + f.write_all(&1000u64.to_be_bytes()).unwrap(); + f.write_all(&[0xAAu8; 24]).unwrap(); + + let patches = patches_for(&f); + + // 16 (header+size64) + 24 payload = 40 bytes remain from the mdat + // start; the 64-bit size field lives at offset 24. + let expected: Vec = 40u64 + .to_be_bytes() + .iter() + .enumerate() + .map(|(i, &b)| Patch { + offset: 24 + i as u64, + value: b, + }) + .collect(); + assert_eq!(patches, expected); + } +} diff --git a/src/fingerprint.rs b/src/fingerprint.rs index e1fdbcb..671a8a0 100644 --- a/src/fingerprint.rs +++ b/src/fingerprint.rs @@ -4,6 +4,7 @@ //! and [chromaprint-next](https://docs.rs/chromaprint-next) for fingerprint generation. //! The fingerprints are bit-identical to those produced by the reference `fpcalc` tool. +use crate::container_repair; use chromaprint::{Algorithm, Fingerprinter}; use std::path::Path; use symphonia::core::audio::{Audio, GenericAudioBufferRef}; @@ -11,7 +12,7 @@ use symphonia::core::codecs::audio::{AudioCodecParameters, AudioDecoderOptions}; use symphonia::core::codecs::CodecParameters; use symphonia::core::formats::probe::Hint; use symphonia::core::formats::{FormatOptions, Track, TrackType}; -use symphonia::core::io::MediaSourceStream; +use symphonia::core::io::{MediaSource, MediaSourceStream}; use symphonia::core::meta::MetadataOptions; /// Maximum duration to fingerprint (in seconds). Chromaprint only needs ~120s. @@ -83,7 +84,21 @@ pub fn fingerprint_file( max_duration_secs: u64, ) -> Result { let file = std::fs::File::open(path).map_err(FingerprintError::OpenFailed)?; - let mss = MediaSourceStream::new(Box::new(file), Default::default()); + let file_len = file.metadata().map_err(FingerprintError::OpenFailed)?.len(); + + // Some files carry container-level defects (e.g. a truncated MP4 whose + // `mdat` size overruns EOF) that symphonia rejects even though the audio + // present decodes fine. Detect them and overlay byte patches as the file + // is read so the available audio can still be fingerprinted. + let patches = container_repair::detect_repairs(&file, path, file_len); + let source: Box = if patches.is_empty() { + Box::new(file) + } else { + Box::new(container_repair::PatchedSource::new( + file, file_len, patches, + )) + }; + let mss = MediaSourceStream::new(source, Default::default()); let mut hint = Hint::new(); if let Some(ext) = path.extension().and_then(|e| e.to_str()) { @@ -142,6 +157,13 @@ pub fn fingerprint_file( { break; } + // A demuxer-level malformed-stream error (e.g. symphonia's isomp4 + // "no atom pending read" on some iTunes-authored .m4a files) means + // the container can't be advanced further. Rather than discarding + // the whole file, stop reading and fingerprint whatever audio we've + // already decoded — this is usually the full track, since the error + // tends to fire on a trailing/unexpected atom after the last packet. + Err(symphonia::core::errors::Error::DecodeError(_)) => break, Err(e) => return Err(FingerprintError::DecodeFailed(e.to_string())), }; diff --git a/src/lib.rs b/src/lib.rs index e29d319..3b1dab1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,6 +27,8 @@ /// AcoustID API client with rate limiting. pub mod acoustid; +/// Detection and in-flight repair of malformed container headers. +pub mod container_repair; /// SQLite-backed fingerprint cache with change detection. pub mod database; /// Execute commands on duplicate files (the non-kept ones).