Add streaming extract parity with p7zip

README.md

@@ -50,6 +50,11 @@ if let Some(fi) = archive.files_info() {
 // Directories are reported as R7zError::Directory; zero-byte files return an empty Vec.
 let data = archive.extract_to_memory(0)?;
 println!("{} bytes", data.len());
+
+// Stream a file directly to any writer.
+let mut out = std::fs::File::create("/tmp/first-file.bin")?;
+let written = archive.extract_to_writer(0, &mut out)?;
+println!("{written} bytes written");
 ```
 
 ### Reading — extract all to disk safely
@@ -58,7 +63,8 @@ println!("{} bytes", data.len());
 archive.extract_all(Path::new("/tmp/output"))?;
 ```
 
-`extract_all` creates directories and zero-byte files correctly and rejects unsafe archive paths.
+`extract_all` creates directories and zero-byte files correctly, rejects unsafe archive paths,
+and streams decoded file data to disk instead of buffering full decoded folders in memory.
 
 ### Reading — encrypted archives
 
@@ -144,6 +150,8 @@ let archive = Archive::from_bytes(raw.into())?;
 | `archive.streams_info()` | `Option<&StreamInfo>` | Raw stream/pack metadata |
 | `archive.extract_to_memory(index: usize)` | `Result<Vec<u8>, R7zError>` | Decompress file at `index` (0-based) |
 | `archive.extract_to_memory_with_password(index, password)` | `Result<Vec<u8>, R7zError>` | Decrypt/decompress file at `index` |
+| `archive.extract_to_writer(index, writer)` | `Result<u64, R7zError>` | Stream file at `index` into a writer |
+| `archive.extract_to_writer_with_password(index, writer, password)` | `Result<u64, R7zError>` | Stream encrypted file data into a writer |
 | `archive.extract_all(dest: &Path)` | `Result<(), R7zError>` | Extract all files; creates subdirectories as needed |
 | `archive.extract_all_with_password(dest, password)` | `Result<(), R7zError>` | Extract all files from an encrypted archive |
 
@@ -284,7 +292,8 @@ Interop tests cover behavioral parity for p7zip-created and r7z-created LZMA,
 LZMA2, and BCJ+x86+LZMA2 archives. The parity target is matching archive
 listing/extraction behavior: file names, file contents, nested paths,
 directories, zero-byte files, and exposed metadata where r7z supports it.
-r7z does not guarantee byte-identical output to p7zip.
+r7z does not guarantee byte-identical archive output, matching compression ratios,
+or matching compressed stream bytes.
 
 LZHAM and Fast LZMA2 variants from p7zip-zstd are not supported.
 

src/aes.rs

@@ -7,11 +7,11 @@
 //!
 //! | Byte | Bits   | Meaning                                        |
 //! |------|--------|------------------------------------------------|
-//! | 0    | [5:0]  | NumCyclesPower (0–62, or 0x3F for raw key)     |
-//! | 0    | [6]    | IV present flag                                |
-//! | 0    | [7]    | Salt present flag                              |
-//! | 1*   | [7:4]  | Extra salt bytes (if salt flag set)            |
-//! | 1*   | [3:0]  | Extra IV bytes (if IV flag set)                |
+//! | 0    | \[5:0\]  | NumCyclesPower (0–62, or 0x3F for raw key)   |
+//! | 0    | \[6\]    | IV present flag                              |
+//! | 0    | \[7\]    | Salt present flag                            |
+//! | 1*   | \[7:4\]  | Extra salt bytes (if salt flag set)          |
+//! | 1*   | \[3:0\]  | Extra IV bytes (if IV flag set)              |
 //! | 2+   |        | Salt bytes, then IV bytes                      |
 //!
 //! \* Byte 1 is only present if either the salt or IV flag is set.
@@ -26,6 +26,9 @@ use sha2::{Digest, Sha256};
 
 type Aes256CbcDec = cbc::Decryptor<Aes256>;
 
+/// Bound p7zip's default AES KDF cost while rejecting maliciously huge values.
+pub(crate) const MAX_AES_NUM_CYCLES_POWER: u8 = 24;
+
 /// Parsed AES-256-SHA-256 properties from a 7z coder.
 #[derive(Debug)]
 pub(crate) struct AesProperties {
@@ -82,7 +85,11 @@ impl AesProperties {
 ///
 /// The password is first encoded as UTF-16LE. Then for `2^num_cycles_power`
 /// iterations, we feed `salt || password_utf16le || counter_le_8bytes` into SHA-256.
-pub(crate) fn derive_key(password: &str, salt: &[u8], num_cycles_power: u8) -> [u8; 32] {
+pub(crate) fn derive_key(
+    password: &str,
+    salt: &[u8],
+    num_cycles_power: u8,
+) -> Result<[u8; 32], R7zError> {
     // Special case: 0x3F means raw key = salt || password, zero-padded
     if num_cycles_power == 0x3F {
         let pwd_utf16: Vec<u8> = password
@@ -93,7 +100,11 @@ pub(crate) fn derive_key(password: &str, salt: &[u8], num_cycles_power: u8) -> [
         let total: Vec<u8> = salt.iter().chain(pwd_utf16.iter()).copied().collect();
         let len = total.len().min(32);
         key[..len].copy_from_slice(&total[..len]);
-        return key;
+        return Ok(key);
+    }
+
+    if num_cycles_power > MAX_AES_NUM_CYCLES_POWER {
+        return Err(R7zError::Decompression);
     }
 
     let pwd_utf16: Vec<u8> = password
@@ -120,7 +131,7 @@ pub(crate) fn derive_key(password: &str, salt: &[u8], num_cycles_power: u8) -> [
     let result = hasher.finalize();
     let mut key = [0u8; 32];
     key.copy_from_slice(&result);
-    key
+    Ok(key)
 }
 
 /// Decrypt `data` using AES-256-CBC with the given key and IV.
@@ -185,7 +196,7 @@ mod tests {
     fn derive_key_known_value() {
         // With 0 cycles (2^0 = 1 iteration), no salt, we can verify manually.
         // SHA256(password_utf16le || 0x0000000000000000)
-        let key = derive_key("a", &[], 0);
+        let key = derive_key("a", &[], 0).unwrap();
         // "a" in UTF-16LE = [0x61, 0x00]
         // One round: SHA256([0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])
         let mut hasher = Sha256::new();
@@ -194,6 +205,14 @@ mod tests {
         assert_eq!(key, expected);
     }
 
+    #[test]
+    fn derive_key_rejects_excessive_cycle_power() {
+        assert!(matches!(
+            derive_key("a", &[], MAX_AES_NUM_CYCLES_POWER + 1),
+            Err(R7zError::Decompression)
+        ));
+    }
+
     #[test]
     fn aes_cbc_decrypt_roundtrip() {
         use aes::Aes256;

src/archive.rs

@@ -4,7 +4,7 @@ use crate::{
 };
 use bytes::Bytes;
 use memmap2::Mmap;
-use std::io::Read;
+use std::io::{BufWriter, Read, Write};
 use std::ops::Range;
 use std::path::{Component, Path, PathBuf};
 
@@ -307,6 +307,49 @@ impl Archive {
         file_index: usize,
         password: Option<&str>,
     ) -> Result<Vec<u8>, R7zError> {
+        let mut bytes = Vec::new();
+        self.extract_to_writer_with_password(file_index, &mut bytes, password)?;
+        Ok(bytes)
+    }
+
+    /// Extract a single file by index into a writer.
+    ///
+    /// This streams the decoded folder into `writer` instead of materializing
+    /// the whole folder in memory. The returned value is the number of file
+    /// bytes written.
+    ///
+    /// # Errors
+    ///
+    /// Returns the same archive, codec, and CRC errors as
+    /// [`extract_to_memory`](Self::extract_to_memory), plus [`R7zError::Io`] for
+    /// writer failures.
+    pub fn extract_to_writer<W: Write + ?Sized>(
+        &self,
+        file_index: usize,
+        writer: &mut W,
+    ) -> Result<u64, R7zError> {
+        self.extract_to_writer_with_password(file_index, writer, None)
+    }
+
+    /// Extract a single file by index into a writer, supplying a password for
+    /// encrypted archives.
+    ///
+    /// The decoder stream is drained after the target file has been written
+    /// whenever a folder CRC is present, so corruption later in the same solid
+    /// block is still detected.
+    ///
+    /// # Errors
+    ///
+    /// Returns [`R7zError::PasswordRequired`] if the file is encrypted and no
+    /// password is supplied, [`R7zError::Crc`] for digest mismatches,
+    /// [`R7zError::Decompression`] for codec failures, or [`R7zError::Io`] for
+    /// writer failures.
+    pub fn extract_to_writer_with_password<W: Write + ?Sized>(
+        &self,
+        file_index: usize,
+        writer: &mut W,
+        password: Option<&str>,
+    ) -> Result<u64, R7zError> {
         if file_index >= self.num_files() {
             return Err(R7zError::Parse);
         }
@@ -316,9 +359,88 @@ impl Archive {
             return Err(R7zError::Directory);
         }
         if fi.is_some_and(|f| f.is_empty_stream(file_index) && f.is_empty_file(file_index)) {
-            return Ok(Vec::new());
+            return Ok(0);
+        }
+
+        let location = self.extraction_location(file_index)?;
+        let packed = &self.data[location.packed_range.clone()];
+        let mut reader = codec::folder_reader(
+            &location.folder,
+            packed,
+            location.folder_unpack_size,
+            password,
+        )?;
+
+        let mut folder_hasher = location.folder_digest.map(|_| crc32fast::Hasher::new());
+        let mut stream_hasher = location.substream_digest.map(|_| crc32fast::Hasher::new());
+        let mut decoded_len = 0u64;
+        let mut remaining_skip = location.stream_start;
+        let mut remaining_take = location.stream_size;
+        let mut written = 0u64;
+        let mut buf = [0u8; 8192];
+
+        loop {
+            let n = reader.read(&mut buf).map_err(|_| R7zError::Decompression)?;
+            if n == 0 {
+                break;
+            }
+
+            decoded_len = decoded_len.checked_add(n as u64).ok_or(R7zError::Parse)?;
+
+            if let Some(hasher) = folder_hasher.as_mut() {
+                hasher.update(&buf[..n]);
+            }
+
+            let mut offset = 0usize;
+            if remaining_skip > 0 {
+                let skip = remaining_skip.min(n);
+                remaining_skip -= skip;
+                offset += skip;
+            }
+
+            if remaining_skip == 0 && remaining_take > 0 && offset < n {
+                let take = remaining_take.min(n - offset);
+                let bytes = &buf[offset..offset + take];
+                writer.write_all(bytes)?;
+                if let Some(hasher) = stream_hasher.as_mut() {
+                    hasher.update(bytes);
+                }
+                remaining_take -= take;
+                written = written.checked_add(take as u64).ok_or(R7zError::Parse)?;
+            }
+
+            if remaining_skip == 0 && remaining_take == 0 && location.folder_digest.is_none() {
+                break;
+            }
+        }
+
+        if remaining_skip > 0 || remaining_take > 0 {
+            return Err(R7zError::Decompression);
+        }
+
+        if let Some(expected) = location.folder_digest {
+            let actual = folder_hasher.ok_or(R7zError::Parse)?.finalize();
+            if actual != expected {
+                return Err(R7zError::Crc);
+            }
+        }
+
+        if let Some(expected) = location.substream_digest {
+            let actual = stream_hasher.ok_or(R7zError::Parse)?.finalize();
+            if actual != expected {
+                return Err(R7zError::Crc);
+            }
+        }
+
+        if decoded_len < location.stream_end_u64()? {
+            return Err(R7zError::Decompression);
         }
 
+        Ok(written)
+    }
+
+    fn extraction_location(&self, file_index: usize) -> Result<ExtractionLocation, R7zError> {
+        let fi = self.header.files_info();
         let streams = self.streams_info().ok_or(R7zError::Parse)?;
         let pack_info = streams.pack_info.as_ref().ok_or(R7zError::Parse)?;
         let unpack_info = streams.unpack_info.as_ref().ok_or(R7zError::Parse)?;
@@ -336,7 +458,7 @@ impl Archive {
         )
         .ok_or(R7zError::Parse)?;
 
-        // Decompress the folder
+        // Locate the packed bytes for the folder that contains this file stream.
         let folder = unpack_info.parse_folder(folder_idx)?;
         let prior_pack_sizes = pack_info
             .pack_size
@@ -350,30 +472,29 @@ impl Archive {
         let pack_pos = usize::try_from(pack_info.pack_pos).map_err(|_| R7zError::Parse)?;
         let data_start = checked_add_usize(checked_add_usize(32, pack_pos)?, pack_offset)?;
         let packed_range = checked_range(self.data.len(), data_start, pack_size)?;
-        let packed = &self.data[packed_range];
 
         let folder_unpack_size = folder_total_unpack_size(folder_idx, unpack_info, substream_info)?;
-        let decompressed =
-            codec::decompress_folder_with_password(&folder, packed, folder_unpack_size, password)?;
-        validate_digest(
-            unpack_info.digests.get(folder_idx).copied().flatten(),
-            &decompressed,
-        )?;
-
-        // Slice the target stream out of the decompressed folder data
         let stream_start =
             stream_offset_in_folder(folder_idx, stream_in_folder, substream_info, unpack_info)?;
         let stream_size =
             stream_size_at(folder_idx, stream_in_folder, substream_info, unpack_info)?;
-        let stream_range = checked_range(decompressed.len(), stream_start, stream_size as u64)?;
-        let extracted = decompressed[stream_range].to_vec();
-
-        if let Some(si) = substream_info {
+        let folder_digest = unpack_info.digests.get(folder_idx).copied().flatten();
+        let substream_digest = if let Some(si) = substream_info {
             let crc_idx = substream_global_index(folder_idx, stream_in_folder, si)?;
-            validate_digest(si.digests.get(crc_idx).copied().flatten(), &extracted)?;
-        }
-
-        Ok(extracted)
+            si.digests.get(crc_idx).copied().flatten()
+        } else {
+            None
+        };
+
+        Ok(ExtractionLocation {
+            folder,
+            packed_range,
+            folder_unpack_size,
+            stream_start,
+            stream_size,
+            folder_digest,
+            substream_digest,
+        })
     }
 
     /// Extract all files to a directory.
@@ -422,11 +543,13 @@ impl Archive {
                 }
                 std::fs::File::create(&dest_path)?;
             } else {
-                let bytes = self.extract_to_memory_with_password(i, password)?;
                 if let Some(parent) = dest_path.parent() {
                     std::fs::create_dir_all(parent)?;
                 }
-                std::fs::write(&dest_path, &bytes)?;
+                let file = std::fs::File::create(&dest_path)?;
+                let mut writer = BufWriter::new(file);
+                self.extract_to_writer_with_password(i, &mut writer, password)?;
+                writer.flush()?;
             }
         }
         Ok(())
@@ -435,6 +558,26 @@ impl Archive {
 
 // ── helpers ──────────────────────────────────────────────────────────────────
 
+struct ExtractionLocation {
+    folder: crate::Folder,
+    packed_range: Range<usize>,
+    folder_unpack_size: u64,
+    stream_start: usize,
+    stream_size: usize,
+    folder_digest: Option<u32>,
+    substream_digest: Option<u32>,
+}
+
+impl ExtractionLocation {
+    fn stream_end_u64(&self) -> Result<u64, R7zError> {
+        let end = self
+            .stream_start
+            .checked_add(self.stream_size)
+            .ok_or(R7zError::Parse)?;
+        u64::try_from(end).map_err(|_| R7zError::Parse)
+    }
+}
+
 fn checked_add_usize(lhs: usize, rhs: usize) -> Result<usize, R7zError> {
     lhs.checked_add(rhs).ok_or(R7zError::Parse)
 }
@@ -449,14 +592,6 @@ fn checked_range(total_len: usize, start: usize, len: u64) -> Result<Range<usize
     }
 }
 
-fn validate_digest(expected: Option<u32>, bytes: &[u8]) -> Result<(), R7zError> {
-    if expected.is_some_and(|crc| crc32fast::hash(bytes) != crc) {
-        Err(R7zError::Crc)
-    } else {
-        Ok(())
-    }
-}
-
 fn safe_archive_path(dest: &Path, name: &str) -> Result<Option<PathBuf>, R7zError> {
     if name.is_empty() || has_windows_prefix(name) || has_parent_component(name) {
         return Err(R7zError::UnsafePath(name.to_string()));