Fingerprint truncated MP4/M4A files with overrun mdat size

src/container_repair/mod.rs

@@ -0,0 +1,104 @@
+//! Detection and in-flight repair of malformed audio container headers.
+//!
+//! Some audio files carry container-level defects (e.g. a truncated MP4 whose
+//! `mdat` box size overruns the real end of the file) that make symphonia
+//! reject them even though the audio present decodes fine. Each format-specific
+//! submodule detects one class of defect and describes the fix as byte
+//! [`Patch`]es; [`PatchedSource`] overlays those patches onto the file as it is
+//! read, so the file on disk is never modified.
+//!
+//! To support a new format repair, add a submodule and a match arm in
+//! [`detect_repairs`].
+
+mod mp4;
+
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom};
+use std::path::Path;
+use symphonia::core::io::MediaSource;
+
+/// A single-byte overlay: when reading, the byte at `offset` is replaced by
+/// `value`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Patch {
+    /// Absolute file offset of the byte to replace.
+    pub offset: u64,
+    /// Replacement byte.
+    pub value: u8,
+}
+
+/// Scan `file` (whose format is chosen by `path`'s extension) for known
+/// container defects and return the byte patches that repair them.
+///
+/// Returns an empty vec for well-formed files (the common case), so the caller
+/// decodes the original file untouched. Only small header regions are read,
+/// not the media data, and the file cursor is rewound to the start before
+/// returning.
+pub fn detect_repairs(file: &File, path: &Path, file_len: u64) -> Vec<Patch> {
+    let ext = path
+        .extension()
+        .and_then(|e| e.to_str())
+        .map(|e| e.to_ascii_lowercase());
+
+    let patches = match ext.as_deref() {
+        Some(e) if mp4::EXTENSIONS.contains(&e) => mp4::mdat_size_patches(file, file_len),
+        _ => Vec::new(),
+    };
+
+    // The scan moves the shared file cursor; rewind so the caller reads from
+    // the top. If this fails, the caller's own reads will surface the error.
+    let mut f = file;
+    let _ = f.seek(SeekFrom::Start(0));
+
+    patches
+}
+
+/// A [`MediaSource`] that overlays a handful of single-byte [`Patch`]es onto a
+/// file as it is read, used to repair a malformed container header (see
+/// [`detect_repairs`]). All other bytes pass through unchanged.
+pub struct PatchedSource {
+    inner: File,
+    len: u64,
+    patches: Vec<Patch>,
+}
+
+impl PatchedSource {
+    /// Wrap `inner` (of total size `len`) so that reads see `patches` applied.
+    pub fn new(inner: File, len: u64, patches: Vec<Patch>) -> Self {
+        Self {
+            inner,
+            len,
+            patches,
+        }
+    }
+}
+
+impl Read for PatchedSource {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        let start = self.inner.stream_position()?;
+        let n = self.inner.read(buf)?;
+        let end = start + n as u64;
+        for &Patch { offset, value } in &self.patches {
+            if offset >= start && offset < end {
+                buf[(offset - start) as usize] = value;
+            }
+        }
+        Ok(n)
+    }
+}
+
+impl Seek for PatchedSource {
+    fn seek(&mut self, pos: SeekFrom) -> std::io::Result<u64> {
+        self.inner.seek(pos)
+    }
+}
+
+impl MediaSource for PatchedSource {
+    fn is_seekable(&self) -> bool {
+        true
+    }
+
+    fn byte_len(&self) -> Option<u64> {
+        Some(self.len)
+    }
+}

src/container_repair/mp4.rs

@@ -0,0 +1,158 @@
+//! Repairs for ISO-BMFF (MP4/M4A) container defects.
+
+use super::Patch;
+use std::fs::File;
+use std::io::{Read, Seek, SeekFrom};
+
+/// MP4/M4A extensions that route to symphonia's isomp4 demuxer and can carry a
+/// truncated `mdat`. Raw AAC (`.aac`) is ADTS, not an ISO-BMFF container.
+pub(super) const EXTENSIONS: &[&str] = &["m4a", "m4b", "m4p", "mp4", "mov"];
+
+/// Scan the top-level atoms of an MP4/M4A file for an `mdat` box whose declared
+/// size overruns the end of the file (an interrupted download or copy), and
+/// return the byte patches that rewrite its size field to the number of bytes
+/// actually present. Symphonia's isomp4 demuxer discards such an mdat wholesale
+/// and then fails with "no atom pending read", even though the audio that IS
+/// present decodes fine; with the size repaired it hits a clean EOF at the
+/// truncation point instead.
+///
+/// Returns an empty vec for well-formed files. This only walks the small
+/// top-level atom headers, not the media data. The file cursor is left where
+/// the scan stopped; the caller is responsible for rewinding.
+pub(super) fn mdat_size_patches(file: &File, file_len: u64) -> Vec<Patch> {
+    let mut f = file;
+    let mut pos: u64 = 0;
+    let mut hdr = [0u8; 16];
+    while pos + 8 <= file_len {
+        if f.seek(SeekFrom::Start(pos)).is_err() || f.read_exact(&mut hdr[..8]).is_err() {
+            break;
+        }
+        let size32 = u32::from_be_bytes([hdr[0], hdr[1], hdr[2], hdr[3]]);
+        let atom_type = [hdr[4], hdr[5], hdr[6], hdr[7]];
+
+        // Resolve the atom size and where/how wide its size field is.
+        let (atom_size, size_off, size_is_64) = if size32 == 1 {
+            // 64-bit size stored immediately after the type field.
+            if f.read_exact(&mut hdr[8..16]).is_err() {
+                break;
+            }
+            let size64 = u64::from_be_bytes(hdr[8..16].try_into().unwrap());
+            (size64, pos + 8, true)
+        } else if size32 == 0 {
+            // Size 0 already means "extends to end of file", and nothing can
+            // follow it — no repair needed or possible.
+            break;
+        } else {
+            (size32 as u64, pos, false)
+        };
+
+        if atom_size < 8 {
+            break; // malformed header; give up rather than loop
+        }
+
+        let atom_end = pos.saturating_add(atom_size);
+
+        if atom_type == *b"mdat" && atom_end > file_len {
+            // Rewrite the size field so the box ends exactly at EOF.
+            let corrected = file_len - pos;
+            let bytes_at = |slice: &[u8]| -> Vec<Patch> {
+                slice
+                    .iter()
+                    .enumerate()
+                    .map(|(i, &b)| Patch {
+                        offset: size_off + i as u64,
+                        value: b,
+                    })
+                    .collect()
+            };
+            return if size_is_64 {
+                bytes_at(&corrected.to_be_bytes())
+            } else if corrected <= u32::MAX as u64 {
+                bytes_at(&(corrected as u32).to_be_bytes())
+            } else {
+                Vec::new() // corrected size won't fit a 32-bit field
+            };
+        }
+
+        pos = atom_end;
+    }
+
+    Vec::new()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write;
+
+    /// Build a file with an `ftyp` atom followed by an `mdat` atom whose
+    /// 32-bit size field declares `declared_payload` bytes of media data,
+    /// while only `actual_payload` bytes are written (truncation).
+    fn mp4_file(declared_payload: u32, actual_payload: usize) -> tempfile::NamedTempFile {
+        let mut f = tempfile::NamedTempFile::new().unwrap();
+        f.write_all(&16u32.to_be_bytes()).unwrap();
+        f.write_all(b"ftypM4A \0\0\0\0").unwrap();
+        f.write_all(&(8 + declared_payload).to_be_bytes()).unwrap();
+        f.write_all(b"mdat").unwrap();
+        f.write_all(&vec![0xAAu8; actual_payload]).unwrap();
+        f
+    }
+
+    fn patches_for(f: &tempfile::NamedTempFile) -> Vec<Patch> {
+        let file = f.reopen().unwrap();
+        let len = file.metadata().unwrap().len();
+        mdat_size_patches(&file, len)
+    }
+
+    #[test]
+    fn well_formed_file_needs_no_patches() {
+        let f = mp4_file(100, 100);
+        assert!(patches_for(&f).is_empty());
+    }
+
+    #[test]
+    fn truncated_mdat_size_is_rewritten_to_bytes_on_disk() {
+        // mdat declares 100 payload bytes but only 40 are present.
+        let f = mp4_file(100, 40);
+        let patches = patches_for(&f);
+
+        // The 32-bit size field at offset 16 must now read 8 + 40 = 48.
+        let expected: Vec<Patch> = 48u32
+            .to_be_bytes()
+            .iter()
+            .enumerate()
+            .map(|(i, &b)| Patch {
+                offset: 16 + i as u64,
+                value: b,
+            })
+            .collect();
+        assert_eq!(patches, expected);
+    }
+
+    #[test]
+    fn truncated_64bit_mdat_size_is_rewritten() {
+        // mdat with size32 == 1 and a 64-bit size that overruns the file.
+        let mut f = tempfile::NamedTempFile::new().unwrap();
+        f.write_all(&16u32.to_be_bytes()).unwrap();
+        f.write_all(b"ftypM4A \0\0\0\0").unwrap();
+        f.write_all(&1u32.to_be_bytes()).unwrap();
+        f.write_all(b"mdat").unwrap();
+        f.write_all(&1000u64.to_be_bytes()).unwrap();
+        f.write_all(&[0xAAu8; 24]).unwrap();
+
+        let patches = patches_for(&f);
+
+        // 16 (header+size64) + 24 payload = 40 bytes remain from the mdat
+        // start; the 64-bit size field lives at offset 24.
+        let expected: Vec<Patch> = 40u64
+            .to_be_bytes()
+            .iter()
+            .enumerate()
+            .map(|(i, &b)| Patch {
+                offset: 24 + i as u64,
+                value: b,
+            })
+            .collect();
+        assert_eq!(patches, expected);
+    }
+}

src/fingerprint.rs

@@ -4,14 +4,15 @@
 //! and [chromaprint-next](https://docs.rs/chromaprint-next) for fingerprint generation.
 //! The fingerprints are bit-identical to those produced by the reference `fpcalc` tool.
 
+use crate::container_repair;
 use chromaprint::{Algorithm, Fingerprinter};
 use std::path::Path;
 use symphonia::core::audio::{Audio, GenericAudioBufferRef};
 use symphonia::core::codecs::audio::{AudioCodecParameters, AudioDecoderOptions};
 use symphonia::core::codecs::CodecParameters;
 use symphonia::core::formats::probe::Hint;
 use symphonia::core::formats::{FormatOptions, Track, TrackType};
-use symphonia::core::io::MediaSourceStream;
+use symphonia::core::io::{MediaSource, MediaSourceStream};
 use symphonia::core::meta::MetadataOptions;
 
 /// Maximum duration to fingerprint (in seconds). Chromaprint only needs ~120s.
@@ -83,7 +84,21 @@ pub fn fingerprint_file(
     max_duration_secs: u64,
 ) -> Result<FingerprintResult, FingerprintError> {
     let file = std::fs::File::open(path).map_err(FingerprintError::OpenFailed)?;
-    let mss = MediaSourceStream::new(Box::new(file), Default::default());
+    let file_len = file.metadata().map_err(FingerprintError::OpenFailed)?.len();
+
+    // Some files carry container-level defects (e.g. a truncated MP4 whose
+    // `mdat` size overruns EOF) that symphonia rejects even though the audio
+    // present decodes fine. Detect them and overlay byte patches as the file
+    // is read so the available audio can still be fingerprinted.
+    let patches = container_repair::detect_repairs(&file, path, file_len);
+    let source: Box<dyn MediaSource> = if patches.is_empty() {
+        Box::new(file)
+    } else {
+        Box::new(container_repair::PatchedSource::new(
+            file, file_len, patches,
+        ))
+    };
+    let mss = MediaSourceStream::new(source, Default::default());
 
     let mut hint = Hint::new();
     if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
@@ -142,6 +157,13 @@ pub fn fingerprint_file(
             {
                 break;
             }
+            // A demuxer-level malformed-stream error (e.g. symphonia's isomp4
+            // "no atom pending read" on some iTunes-authored .m4a files) means
+            // the container can't be advanced further. Rather than discarding
+            // the whole file, stop reading and fingerprint whatever audio we've
+            // already decoded — this is usually the full track, since the error
+            // tends to fire on a trailing/unexpected atom after the last packet.
+            Err(symphonia::core::errors::Error::DecodeError(_)) => break,
             Err(e) => return Err(FingerprintError::DecodeFailed(e.to_string())),
         };
 

src/lib.rs

@@ -27,6 +27,8 @@
 
 /// AcoustID API client with rate limiting.
 pub mod acoustid;
+/// Detection and in-flight repair of malformed container headers.
+pub mod container_repair;
 /// SQLite-backed fingerprint cache with change detection.
 pub mod database;
 /// Execute commands on duplicate files (the non-kept ones).