diff --git a/Cargo.toml b/Cargo.toml index 23635e8..aa7ad3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ thiserror = "2" aes = "0.8" cbc = "0.1" sha2 = "0.10" +getrandom = "0.3" [dev-dependencies] criterion = { version = "0.8", features = ["html_reports"] } diff --git a/README.md b/README.md index 40edc4d..0ca6714 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,11 @@ let written = archive.extract_to_writer(0, &mut out)?; println!("{written} bytes written"); ``` +`Archive::open` is file-backed and uses `mmap` by default, so opening a large +archive does not allocate a heap buffer for the full file. Use +`ArchiveOpenOptions { storage_mode: ArchiveStorageMode::Seek, ..Default::default() }` +when mmap is undesirable. + ### Reading — extract all to disk safely ```rust @@ -76,7 +81,7 @@ let archive = Archive::open_with_password(Path::new("secret.7z"), Some("passphra let data = archive.extract_to_memory_with_password(0, Some("passphrase"))?; ``` -### Building a single-file LZMA archive +### Building a single-file LZMA2 archive ```rust use r7z::ArchiveBuilder; @@ -87,18 +92,42 @@ let bytes = ArchiveBuilder::new() std::fs::write("out.7z", &bytes)?; ``` -### Building a multi-file LZMA2 archive +### Building a multi-file archive with explicit entries ```rust -use r7z::{ArchiveBuilder, Codec}; +use r7z::{ArchiveBuilder, EntryMeta}; let bytes = ArchiveBuilder::new() .add_file("alpha.txt", b"AAAA") + .add_empty_file("empty.txt", EntryMeta::default()) + .add_directory("beta", EntryMeta::default()) .add_file("beta/beta.txt", b"BBBBBBBB") - .compression(Codec::Lzma2) .build()?; ``` +### Writing metadata and encoded headers + +```rust +use r7z::{ArchiveBuilder, ArchiveOptions, EntryMeta, HeaderMode}; +use std::time::{Duration, UNIX_EPOCH}; + +let meta = EntryMeta { + mtime: Some(UNIX_EPOCH + Duration::from_secs(1_710_504_000)), + start_pos: Some(0), + ..EntryMeta::from_unix_mode(0o100_644) +}; + +let bytes = ArchiveBuilder::new() + .options(ArchiveOptions { + header_mode: HeaderMode::Encoded, + ..ArchiveOptions::default() + }) + .add_file_entry("metadata.txt", b"with metadata", meta) + .build()?; +``` + +`EntryMeta::attributes` stores raw 7z `WinAttrib` values. `EntryMeta::from_unix_mode(mode)` sets `(mode << 16) | 0x20`, `EntryMeta::directory_unix_mode(mode)` sets `(mode << 16) | 0x10`, and `EntryMeta::archive_file()` sets `0x20`. + ### Building a BCJ+x86+LZMA2 archive ```rust @@ -110,7 +139,29 @@ let bytes = ArchiveBuilder::new() .build()?; ``` -### Streaming builder — process large archives without loading all data into memory +### Building Copy or AES-encrypted archives + +```rust +use r7z::{ArchiveBuilder, ArchiveOptions, Codec, EncryptionOptions}; + +let copy_bytes = ArchiveBuilder::new() + .compression(Codec::Copy) + .add_file("stored.bin", b"stored without compression") + .build()?; + +let mut options = ArchiveOptions::default(); +options.encryption = Some(EncryptionOptions::default_for_password("secret")); + +let encrypted_bytes = ArchiveBuilder::new() + .options(options) + .add_file("secret.txt", b"encrypted content") + .build()?; +``` + +Set `EncryptionOptions::encrypt_header = true` to hide filenames and metadata until the password is supplied. +`EncryptionOptions::default_for_password(password)` uses p7zip-compatible writer defaults: cycle power 19, no salt, and a random 16-byte IV. Non-default salt and IV lengths up to 16 bytes are also supported. + +### Writer API — file-backed archive creation ```rust use r7z::build_streaming; @@ -123,10 +174,36 @@ let entries = vec![ ("file2.bin".to_string(), File::open("file2.bin")?), ].into_iter(); -build_streaming(entries, out_file)?; // Pipes files through compressor directly to disk +build_streaming(entries, out_file)?; +``` + +### Parsing from a seekable reader or in-memory buffer + +7z archives require random access. `Archive::from_reader` accepts `Read + Seek` +sources such as `std::io::Cursor>` or `std::fs::File`; non-seekable +streams should be spooled by the caller before opening. + +```rust +let file = std::fs::File::open("example.7z")?; +let archive = Archive::from_reader(file)?; +``` + +`ArchiveOpenOptions` controls file-backed storage mode and metadata limits: + +```rust +use r7z::{Archive, ArchiveOpenOptions, ArchiveStorageMode}; +use std::path::Path; + +let archive = Archive::open_with_options( + Path::new("example.7z"), + ArchiveOpenOptions { + storage_mode: ArchiveStorageMode::Seek, + max_metadata_bytes: 64 * 1024 * 1024, + }, +)?; ``` -### Parsing from an in-memory buffer +For explicitly in-memory archives, use `Archive::from_bytes`: ```rust let raw: Vec = std::fs::read("example.7z")?; @@ -139,10 +216,11 @@ let archive = Archive::from_bytes(raw.into())?; | Method | Returns | Description | |--------|---------|-------------| -| `Archive::open(path: &Path)` | `Result` | Read and fully decode a `.7z` file from disk | +| `Archive::open(path: &Path)` | `Result` | File-backed open using mmap by default | | `Archive::open_with_password(path, password)` | `Result` | Open an archive with encrypted headers | -| `Archive::from_reader(reader)` | `Result` | Buffer and decode any `Read` source | -| `Archive::from_reader_with_password(reader, password)` | `Result` | Buffer and decode a password-protected `Read` source | +| `Archive::open_with_options(path, options)` | `Result` | Open with mmap/seek storage and metadata limits | +| `Archive::from_reader(reader)` | `Result` | Decode a seekable `Read + Seek` source | +| `Archive::from_reader_with_password(reader, password)` | `Result` | Decode a password-protected seekable source | | `Archive::from_bytes(data: bytes::Bytes)` | `Result` | Decode a `.7z` from an in-memory buffer | | `Archive::from_bytes_with_password(data, password)` | `Result` | Decode password-protected bytes | | `archive.num_files()` | `usize` | Number of entries (files and directories) | @@ -172,30 +250,42 @@ Builder pattern — all methods consume `self` and return `Self` for chaining: | Method | Description | |--------|-------------| -| `ArchiveBuilder::new()` | Create an empty builder (LZMA compression default) | +| `ArchiveBuilder::new()` | Create an empty builder (LZMA2 compression default) | | `.add_file(name: &str, data: &[u8])` | Queue a file with its content | -| `.compression(codec: Codec)` | Set compression (`Codec::Lzma` or `Codec::Lzma2`) | +| `.add_symlink(name, target, meta)` | Queue a symlink-like entry; target bytes are stored as file data and Unix symlink mode bits are set | +| `.add_entry(entry, data)` | Queue an explicit `ArchiveEntry`; non-file entries must not provide stream data | +| `.add_empty_file(name, meta)` / `.add_directory(name, meta)` / `.add_anti_item(name, meta)` | Queue empty-stream entries | +| `.compression(codec: Codec)` | Set compression (`Codec::Copy`, `Codec::Lzma`, `Codec::Lzma2`, or `Codec::Lzma2Bcj`) | +| `.options(options: ArchiveOptions)` | Set codec, header mode, encryption, compression tuning, and streaming options | | `.build()` | Produce the final `.7z` bytes as `Result, R7zError>` | -The builder uses **solid compression**: all files are concatenated into one stream before compressing, which gives better ratios for many small files. +The builder defaults to **LZMA2**, matching p7zip / 7-Zip create behavior. It uses **solid compression** for non-empty files: file data is concatenated into one stream before compression, while directories, anti-items, and zero-byte files are represented with 7z empty-stream metadata. -### `ArchiveWriter` and `build_streaming` — Streaming builders for large archives +`ArchiveOptions::compression` exposes p7zip-like tuning through `CompressionOptions`: +`CompressionLevel`, optional dictionary size, optional fast bytes, `SolidMode` +(`Solid`, `NonSolid`, or `Limit`), and optional LZMA2 chunk size. The existing +`Codec` still selects the algorithm. -`ArchiveWriter` writes one or more compression folders and can store optional per-entry metadata: +### `ArchiveWriter` and `build_streaming` — file-backed builders + +`ArchiveWriter` writes one or more compression folders and can store optional per-entry metadata. It also accepts explicit `ArchiveEntry` values through `.append_archive_entry(...)` and `.append_empty_entry(...)`: ```rust -use r7z::{ArchiveWriter, Codec, EntryMeta}; +use r7z::{ArchiveOptions, ArchiveWriter, Codec, EntryMeta}; use std::fs::File; let file = File::create("out.7z")?; -let mut writer = ArchiveWriter::new(file)?.compression(Codec::Lzma2); -writer.append("a.txt", &mut b"hello".as_ref())?; +let mut writer = ArchiveWriter::new(file, ArchiveOptions::default())?.compression(Codec::Lzma2); +writer.append_file("a.txt", &mut b"hello".as_ref(), EntryMeta::default())?; +writer.append_empty_file("empty.txt", EntryMeta::default())?; writer.new_folder()?; writer.append_entry("b.txt", &mut b"world".as_ref(), EntryMeta::default())?; writer.finish()?; ``` -For archives too large to fit in memory, use the streaming builder: +When configured with `Codec::Copy` and no encryption, `ArchiveWriter` writes non-empty file payloads directly to the output as they are appended. With `Codec::Lzma`, default `Codec::Lzma2`, or `Codec::Lzma2Bcj` and no encryption, it streams into the compressed folder and writes those bytes when the folder is sealed by `new_folder()` or `finish()`. Encrypted writer paths still collect input before final archive assembly. + +For file-backed output, use the convenience builder: ```rust pub fn build_streaming(entries: I, out: W) -> Result<(), R7zError> @@ -203,21 +293,66 @@ where W: Write + Seek, I: IntoIterator, R: Read, + +pub fn build_streaming_with_options( + entries: I, + out: W, + options: ArchiveOptions +) -> Result<(), R7zError> +where + W: Write + Seek, + I: IntoIterator, + R: Read, + +pub fn build_streaming_to_writer( + entries: I, + out: W, + options: ArchiveOptions +) -> Result<(), R7zError> +where + W: Write, + I: IntoIterator, + R: Read, + +pub fn build_streaming_volumes( + entries: I, + base_path: P, + archive_options: ArchiveOptions, + volume_options: VolumeOptions +) -> Result, R7zError> +where + P: AsRef, + I: IntoIterator, + R: Read, ``` -Each `entry` (filename, `impl Read`) is piped through the LZMA2 compressor directly to the output file. Neither all input data nor all compressed output is held in memory simultaneously — only one file at a time. +Each `entry` is provided as a filename and `impl Read`; the builder writes the final `.7z` archive to any `Write + Seek` output. Use `build_streaming_with_options` for Copy, explicit header mode, or encryption settings. +`build_streaming_to_writer` accepts plain `Write` sinks by assembling through an +internal spool first. `build_streaming_volumes` writes p7zip-style split output +such as `archive.7z.001`, `archive.7z.002`, and returns the created paths. + +### Link metadata + +`FilesInfo::entry_type(index)` classifies entries as `File`, `Directory`, +`EmptyFile`, `Anti`, or `Symlink`. `Archive::symlink_target(index)` returns the +stored symlink target for entries marked with Unix symlink mode bits. `extract_all` +does not create filesystem symlinks; symlink entries extract as regular files +containing the target path bytes. Hard-link preservation is not supported for +`.7z` because p7zip does not reliably emit a standard hard-link representation +for this format. ### `Codec` — Compression algorithms ```rust pub enum Codec { - Lzma, // Classic LZMA — codec ID [0x03, 0x01, 0x01] - Lzma2, // Modern LZMA2 — codec ID [0x21] + Copy, // No compression — codec ID [0x00] + Lzma, // Classic LZMA — codec ID [0x03, 0x01, 0x01] + Lzma2, // Default — codec ID [0x21] Lzma2Bcj, // x86 BCJ filter followed by LZMA2 } ``` -`Lzma2` is the p7zip default and generally gives slightly better compression ratios. +`Lzma2` is the default and generally gives slightly better compression ratios. ### `R7zError` — Error variants @@ -233,6 +368,7 @@ pub enum Codec { | `R7zError::WrongPassword` | Reserved for password-specific failures | | `R7zError::UnsafePath(String)` | Extracted path would escape the destination | | `R7zError::Directory` | Requested entry is a directory or anti-item | +| `R7zError::LimitExceeded(&'static str)` | Configured metadata or safety limit was exceeded | **Error handling example:** @@ -270,23 +406,31 @@ These are public but primarily used for building advanced tooling: | Feature | Status | |---------|--------| | LZMA compression | Read + Write | -| LZMA2 compression | Read + Write | -| Copy codec | Read | +| LZMA2 compression | Read + Write (default) | +| Copy codec | Read + Write | | BCJ x86 filter + LZMA2 | Read + Write | -| Uncompressed Copy codec | Read | -| EncodedHeader archives (p7zip default) | Read | +| EncodedHeader archives (p7zip default) | Read + Write | | Uncompressed Header archives | Read + Write | | Solid archives | Read + Write | | Multi-file archives | Read + Write | | Multi-folder / non-solid archives | Read + Write via `ArchiveWriter` | -| AES-256-SHA-256 encrypted content | Read | -| AES encrypted headers (`-mhe=on`) | Read with password | +| Directories / zero-byte files / anti-items | Read + Write | +| AES-256-SHA-256 encrypted content | Read + Write | +| AES encrypted headers (`-mhe=on`) | Read + Write with password | +| Update existing archives | Not supported | | Deflate / BZip2 / PPMd | Not supported | -| AES writing | Not supported | +| Read split volumes | Not supported | +| Hard-link preservation | Not supported | **7z specification:** [7zFormat.txt](https://github.com/google/omaha/blob/master/third_party/lzma/files/7zFormat.txt) -Archives written by r7z use format version 0.4 (standard), are in uncompressed-Header format, and are fully readable by 7-Zip ≥ 9.x and p7zip. +Archives written by r7z use format version 0.4 (standard). Multi-entry archives use EncodedHeader by default, matching p7zip behavior, and are fully readable by 7-Zip ≥ 9.x and p7zip. + +`extract_to_writer` streams decoded file data into the supplied writer and is +the preferred low-allocation extraction API. `extract_to_memory` intentionally +allocates the requested file contents. AES-encrypted extraction still buffers +the encrypted pack stream internally before AES-CBC decryption; streaming AES is +a planned hardening follow-up. Interop tests cover behavioral parity for p7zip-created and r7z-created LZMA, LZMA2, and BCJ+x86+LZMA2 archives. The parity target is matching archive @@ -312,7 +456,7 @@ This loads the dev shell with: - **Profiling**: `perf`, `cargo-flamegraph`, `valgrind` - **Build**: `cargo-nextest`, `gnuplot`, `hyperfine` -Running `cargo flamegraph --bin build_n64 -- /mnt/emulation/n64 /tmp/n64_build.7z` will build a streaming 7z archive from a directory tree and profile the codepath. +Running `cargo flamegraph --bin build_n64 -- /mnt/emulation/n64 /tmp/n64_build.7z` will build a 7z archive from a directory tree and profile the codepath. ### Without Nix diff --git a/benches/parse_bench.rs b/benches/parse_bench.rs index af7adcd..2706cd4 100644 --- a/benches/parse_bench.rs +++ b/benches/parse_bench.rs @@ -1,8 +1,10 @@ #![allow(clippy::semicolon_if_nothing_returned)] use criterion::{criterion_group, criterion_main, Criterion}; +use std::fs::File; use std::hint::black_box; -use std::path::Path; +use std::io::Write; +use std::path::{Path, PathBuf}; fn fixture_bytes() -> Vec { std::fs::read("tests/fixtures/test_1.7z").expect("test fixture missing") @@ -15,16 +17,65 @@ fn bench_signature_parse(c: &mut Criterion) { }); } -fn bench_archive_open(c: &mut Criterion) { +fn bench_archive_open_mmap(c: &mut Criterion) { let path = Path::new("tests/fixtures/test_1.7z"); - c.bench_function("Archive::open", |b| { + c.bench_function("Archive::open mmap fixture", |b| { b.iter(|| r7z::Archive::open(black_box(path)).unwrap()) }); } +fn bench_archive_open_seek(c: &mut Criterion) { + let path = Path::new("tests/fixtures/test_1.7z"); + c.bench_function("Archive::open seek fixture", |b| { + b.iter(|| { + r7z::Archive::open_with_options( + black_box(path), + r7z::ArchiveOpenOptions { + storage_mode: r7z::ArchiveStorageMode::Seek, + ..Default::default() + }, + ) + .unwrap(); + }) + }); +} + +fn sparse_archive_path() -> PathBuf { + let fixture_dir = PathBuf::from("target/bench-fixtures"); + std::fs::create_dir_all(&fixture_dir).expect("create bench fixture dir"); + let path = fixture_dir.join("sparse_256mb.7z"); + if !path.exists() { + let bytes = r7z::ArchiveBuilder::new() + .add_file("payload.txt", b"sparse") + .build() + .expect("build sparse archive fixture"); + let mut file = File::create(&path).expect("create sparse archive fixture"); + file.write_all(&bytes).expect("write sparse archive header"); + file.set_len(256 * 1024 * 1024) + .expect("extend sparse archive fixture"); + } + path +} + +fn bench_archive_open_seek_sparse(c: &mut Criterion) { + let path = sparse_archive_path(); + c.bench_function("Archive::open seek sparse_256mb", |b| { + b.iter(|| { + r7z::Archive::open_with_options( + black_box(path.as_path()), + r7z::ArchiveOpenOptions { + storage_mode: r7z::ArchiveStorageMode::Seek, + ..Default::default() + }, + ) + .unwrap(); + }) + }); +} + fn bench_archive_from_bytes(c: &mut Criterion) { let data: bytes::Bytes = fixture_bytes().into(); - c.bench_function("Archive::from_bytes", |b| { + c.bench_function("Archive::from_bytes fixture", |b| { b.iter(|| r7z::Archive::from_bytes(black_box(data.clone())).unwrap()) }); } @@ -42,6 +93,30 @@ fn bench_extract_to_memory(c: &mut Criterion) { }); } +fn bench_extract_to_writer_seek_backed(c: &mut Criterion) { + let archive = r7z::Archive::open_with_options( + Path::new("tests/fixtures/test_1.7z"), + r7z::ArchiveOpenOptions { + storage_mode: r7z::ArchiveStorageMode::Seek, + ..Default::default() + }, + ) + .unwrap(); + let fi = archive.files_info().unwrap(); + let num_files = usize::try_from(fi.num_files).expect("num_files fits in usize"); + let idx = (0..num_files) + .find(|&i| !fi.is_empty_stream(i)) + .unwrap_or(0); + c.bench_function("Archive::extract_to_writer seek-backed non-aes", |b| { + b.iter(|| { + let mut sink = std::io::sink(); + archive + .extract_to_writer(black_box(idx), &mut sink) + .unwrap(); + }) + }); +} + fn bench_builder_single_file(c: &mut Criterion) { let data = b"Hello, benchmark world! This is a typical short file."; c.bench_function("ArchiveBuilder::build (single file)", |b| { @@ -57,9 +132,12 @@ fn bench_builder_single_file(c: &mut Criterion) { criterion_group!( benches, bench_signature_parse, - bench_archive_open, + bench_archive_open_mmap, + bench_archive_open_seek, + bench_archive_open_seek_sparse, bench_archive_from_bytes, bench_extract_to_memory, + bench_extract_to_writer_seek_backed, bench_builder_single_file, ); criterion_main!(benches); diff --git a/src/aes.rs b/src/aes.rs index 3a24c61..2127d3d 100644 --- a/src/aes.rs +++ b/src/aes.rs @@ -7,7 +7,7 @@ //! //! | Byte | Bits | Meaning | //! |------|--------|------------------------------------------------| -//! | 0 | \[5:0\] | NumCyclesPower (0–62, or 0x3F for raw key) | +//! | 0 | \[5:0\] | `NumCyclesPower` (0–62, or 0x3F for raw key) | //! | 0 | \[6\] | IV present flag | //! | 0 | \[7\] | Salt present flag | //! | 1* | \[7:4\] | Extra salt bytes (if salt flag set) | @@ -16,15 +16,16 @@ //! //! \* Byte 1 is only present if either the salt or IV flag is set. //! -//! Salt size = ((byte0 >> 7) & 1) + (byte1 >> 4) +//! Salt size = ((byte0 >> 7) & 1) + (byte1 >> 4) //! IV size = ((byte0 >> 6) & 1) + (byte1 & 0x0F) use crate::R7zError; use aes::Aes256; -use cbc::cipher::{BlockDecryptMut, KeyIvInit}; +use cbc::cipher::{BlockDecryptMut, BlockEncryptMut, KeyIvInit}; use sha2::{Digest, Sha256}; type Aes256CbcDec = cbc::Decryptor; +type Aes256CbcEnc = cbc::Encryptor; /// Bound p7zip's default AES KDF cost while rejecting maliciously huge values. pub(crate) const MAX_AES_NUM_CYCLES_POWER: u8 = 24; @@ -32,7 +33,7 @@ pub(crate) const MAX_AES_NUM_CYCLES_POWER: u8 = 24; /// Parsed AES-256-SHA-256 properties from a 7z coder. #[derive(Debug)] pub(crate) struct AesProperties { - /// Number of SHA-256 iterations = 2^num_cycles_power. + /// Number of SHA-256 iterations = `2^num_cycles_power`. pub num_cycles_power: u8, /// Salt (0..16 bytes). pub salt: Vec, @@ -92,10 +93,7 @@ pub(crate) fn derive_key( ) -> Result<[u8; 32], R7zError> { // Special case: 0x3F means raw key = salt || password, zero-padded if num_cycles_power == 0x3F { - let pwd_utf16: Vec = password - .encode_utf16() - .flat_map(|c| c.to_le_bytes()) - .collect(); + let pwd_utf16: Vec = password.encode_utf16().flat_map(u16::to_le_bytes).collect(); let mut key = [0u8; 32]; let total: Vec = salt.iter().chain(pwd_utf16.iter()).copied().collect(); let len = total.len().min(32); @@ -107,10 +105,7 @@ pub(crate) fn derive_key( return Err(R7zError::Decompression); } - let pwd_utf16: Vec = password - .encode_utf16() - .flat_map(|c| c.to_le_bytes()) - .collect(); + let pwd_utf16: Vec = password.encode_utf16().flat_map(u16::to_le_bytes).collect(); let num_rounds: u64 = 1u64 << num_cycles_power; @@ -158,6 +153,43 @@ pub(crate) fn decrypt_aes256_cbc( Ok(buf) } +pub(crate) fn encode_aes_properties(num_cycles_power: u8, salt: &[u8], iv: &[u8]) -> Vec { + assert!(salt.len() <= 16, "7z AES salt must be at most 16 bytes"); + assert!(iv.len() <= 16, "7z AES IV must be at most 16 bytes"); + let mut props = Vec::with_capacity(2 + salt.len() + iv.len()); + let has_salt = !salt.is_empty(); + let has_iv = !iv.is_empty(); + props.push( + (num_cycles_power & 0x3F) | if has_salt { 0x80 } else { 0 } | if has_iv { 0x40 } else { 0 }, + ); + if has_salt || has_iv { + let salt_extra = salt.len().saturating_sub(usize::from(has_salt)); + let iv_extra = iv.len().saturating_sub(usize::from(has_iv)); + let salt_extra = u8::try_from(salt_extra).expect("salt length checked"); + let iv_extra = u8::try_from(iv_extra).expect("IV length checked"); + props.push((salt_extra << 4) | iv_extra); + props.extend_from_slice(salt); + props.extend_from_slice(iv); + } + props +} + +pub(crate) fn encrypt_aes256_cbc_zero_pad( + data: &[u8], + key: &[u8; 32], + iv: &[u8; 16], +) -> Result, R7zError> { + let padded_len = data.len().next_multiple_of(16); + let padded_len = padded_len.max(16); + let mut buf = vec![0u8; padded_len]; + buf[..data.len()].copy_from_slice(data); + let encryptor = Aes256CbcEnc::new(key.into(), iv.into()); + let out = encryptor + .encrypt_padded_mut::(&mut buf, padded_len) + .map_err(|_| R7zError::Decompression)?; + Ok(out.to_vec()) +} + #[cfg(test)] mod tests { use super::*; @@ -233,4 +265,28 @@ mod tests { let decrypted = decrypt_aes256_cbc(&ciphertext, &key, &iv).unwrap(); assert_eq!(&decrypted, plaintext); } + + #[test] + fn encode_aes_properties_parse_roundtrip() { + let salt = [0xAA, 0xBB, 0xCC, 0xDD]; + let iv = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let props = encode_aes_properties(19, &salt, &iv); + let parsed = AesProperties::parse(&props).unwrap(); + assert_eq!(parsed.num_cycles_power, 19); + assert_eq!(parsed.salt, salt); + assert_eq!(parsed.iv, iv); + } + + #[test] + fn encrypt_zero_pad_decrypt_truncates_to_original() { + let key = [0x33u8; 32]; + let iv = [0x44u8; 16]; + let plaintext = b"not a block multiple"; + let encrypted = encrypt_aes256_cbc_zero_pad(plaintext, &key, &iv).unwrap(); + assert!(encrypted.len().is_multiple_of(16)); + + let mut decrypted = decrypt_aes256_cbc(&encrypted, &key, &iv).unwrap(); + decrypted.truncate(plaintext.len()); + assert_eq!(decrypted, plaintext); + } } diff --git a/src/archive.rs b/src/archive.rs index 700abb7..dc5d5ee 100644 --- a/src/archive.rs +++ b/src/archive.rs @@ -4,15 +4,139 @@ use crate::{ }; use bytes::Bytes; use memmap2::Mmap; -use std::io::{BufWriter, Read, Write}; +use std::io::{BufWriter, Read, Seek, SeekFrom, Write}; use std::ops::Range; use std::path::{Component, Path, PathBuf}; +use std::sync::Mutex; /// Maximum decompressed size accepted for the compressed archive header (metadata only). /// A malicious archive could declare an enormous `unpack_size` to cause OOM during header /// decompression; this cap bounds the allocation to a sane limit. File data extracted /// via [`Archive::extract_to_memory`] is not subject to this limit. -const MAX_HEADER_UNPACK_BYTES: u64 = 64 * 1024 * 1024; +const DEFAULT_MAX_METADATA_BYTES: u64 = 64 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ArchiveStorageMode { + Mmap, + Seek, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ArchiveOpenOptions { + pub max_metadata_bytes: u64, + pub storage_mode: ArchiveStorageMode, +} + +impl Default for ArchiveOpenOptions { + fn default() -> Self { + Self { + max_metadata_bytes: DEFAULT_MAX_METADATA_BYTES, + storage_mode: ArchiveStorageMode::Mmap, + } + } +} + +trait ReadSeek: Read + Seek {} + +impl ReadSeek for T {} + +enum ArchiveSource { + Bytes(Bytes), + Seekable { + reader: Mutex>, + len: u64, + }, +} + +impl ArchiveSource { + fn from_reader(mut reader: R) -> Result + where + R: Read + Seek + Send + 'static, + { + let len = reader.seek(SeekFrom::End(0)).map_err(R7zError::Io)?; + Ok(Self::Seekable { + reader: Mutex::new(Box::new(reader)), + len, + }) + } + + fn len(&self) -> Result { + match self { + Self::Bytes(bytes) => u64::try_from(bytes.len()).map_err(|_| R7zError::Parse), + Self::Seekable { len, .. } => Ok(*len), + } + } + + fn read_exact_at(&self, offset: u64, dst: &mut [u8]) -> Result<(), R7zError> { + if dst.is_empty() { + return Ok(()); + } + let end = offset + .checked_add(u64::try_from(dst.len()).map_err(|_| R7zError::Parse)?) + .ok_or(R7zError::Parse)?; + if end > self.len()? { + return Err(R7zError::Parse); + } + match self { + Self::Bytes(bytes) => { + let start = usize::try_from(offset).map_err(|_| R7zError::Parse)?; + let end = usize::try_from(end).map_err(|_| R7zError::Parse)?; + dst.copy_from_slice(bytes.get(start..end).ok_or(R7zError::Parse)?); + Ok(()) + } + Self::Seekable { reader, .. } => { + let mut reader = reader.lock().map_err(|_| R7zError::Parse)?; + reader.seek(SeekFrom::Start(offset))?; + reader.read_exact(dst)?; + Ok(()) + } + } + } + + fn read_range_to_vec(&self, range: Range, limit: u64) -> Result, R7zError> { + let len = checked_sub_u64(range.end, range.start)?; + if len > limit { + return Err(R7zError::LimitExceeded("metadata")); + } + let len = usize::try_from(len).map_err(|_| R7zError::Parse)?; + let mut out = vec![0u8; len]; + self.read_exact_at(range.start, &mut out)?; + Ok(out) + } + + fn range_reader(&self, range: Range) -> Result, R7zError> { + if range.start > range.end || range.end > self.len()? { + return Err(R7zError::Parse); + } + Ok(ArchiveRangeReader { + source: self, + pos: range.start, + end: range.end, + }) + } +} + +struct ArchiveRangeReader<'a> { + source: &'a ArchiveSource, + pos: u64, + end: u64, +} + +impl Read for ArchiveRangeReader<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if self.pos >= self.end || buf.is_empty() { + return Ok(0); + } + let remaining = self.end - self.pos; + let n = usize::try_from(remaining.min(buf.len() as u64)) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "range too large"))?; + self.source + .read_exact_at(self.pos, &mut buf[..n]) + .map_err(std::io::Error::other)?; + self.pos += n as u64; + Ok(n) + } +} /// Metadata extracted from the outer 7z header (`EncodedHeader` only). #[derive(Debug)] @@ -62,8 +186,7 @@ impl ArchiveMetadata { /// Fully decoded archive with file listing and extraction support. pub struct Archive { - /// Raw archive bytes (O(1) clone via reference counting). - data: Bytes, + source: ArchiveSource, pub signature: SignatureHeader, /// Present for `EncodedHeader` archives; None for uncompressed-header archives. pub encoded_header: Option, @@ -89,7 +212,7 @@ impl Archive { /// this is rarely an issue for archive files, but callers that need /// stronger guarantees should use [`Archive::from_reader`] instead. pub fn open(path: &Path) -> Result { - Self::open_with_password(path, None) + Self::open_with_options(path, ArchiveOpenOptions::default()) } /// Open a 7z archive from a file path, supplying a password for encrypted archives. @@ -104,40 +227,96 @@ impl Archive { /// [`R7zError::PasswordRequired`] if the headers are encrypted and no password /// is supplied, or a parse/CRC error if the archive is malformed. pub fn open_with_password(path: &Path, password: Option<&str>) -> Result { + Self::open_with_password_and_options(path, password, ArchiveOpenOptions::default()) + } + + pub fn open_with_options( + path: &Path, + options: ArchiveOpenOptions, + ) -> Result { + Self::open_with_password_and_options(path, None, options) + } + + pub fn open_with_password_and_options( + path: &Path, + password: Option<&str>, + options: ArchiveOpenOptions, + ) -> Result { let file = std::fs::File::open(path)?; - // SAFETY: The file is opened read-only and we do not mutate the mapping. - // A concurrent truncation of the file could cause SIGBUS; callers that - // need to guard against this should use `from_reader` instead. - let mmap = unsafe { Mmap::map(&file)? }; - Self::from_bytes_with_password(Bytes::from_owner(mmap), password) + let source = match options.storage_mode { + ArchiveStorageMode::Mmap => { + // SAFETY: The file is opened read-only and we do not mutate the + // mapping. A concurrent truncation could cause SIGBUS; callers + // that need stronger guarantees can select Seek mode. + let mmap = unsafe { Mmap::map(&file)? }; + ArchiveSource::Bytes(Bytes::from_owner(mmap)) + } + ArchiveStorageMode::Seek => ArchiveSource::from_reader(file)?, + }; + Self::from_source_with_password(source, password, options) } - /// Fully read a [`Read`] source and decode it as a 7z archive. + /// Decode a seekable [`Read`] source as a 7z archive. /// - /// Because the 7z format requires random access (the header lives at the - /// end of the file while the data blocks are near the start), the entire - /// source is buffered into memory before parsing begins. For local files - /// prefer [`Archive::open`], which uses `mmap` to avoid an upfront - /// allocation. + /// The 7z format needs random access: packed streams are near the start, + /// while the authoritative header is usually near the end. Non-seekable + /// sources must be spooled by the caller before constructing an [`Archive`]. /// /// # Errors /// /// Returns [`R7zError::Io`] if reading fails, or a parse/CRC error if the /// archive is malformed. - pub fn from_reader(reader: impl Read) -> Result { + pub fn from_reader(reader: R) -> Result + where + R: Read + Seek + Send + 'static, + { Self::from_reader_with_password(reader, None) } - /// Fully read a [`Read`] source and decode it as a 7z archive, with a password. + /// Decode a seekable [`Read`] source as a 7z archive, with a password. /// /// See [`Archive::from_reader`] and [`Archive::open_with_password`] for details. - pub fn from_reader_with_password( - mut reader: impl Read, + /// + /// # Errors + /// + /// Returns [`R7zError::Io`] if reading fails, [`R7zError::PasswordRequired`] + /// if encrypted headers need a password, or a parse/CRC error if malformed. + pub fn from_reader_with_password( + reader: R, password: Option<&str>, - ) -> Result { - let mut buf = Vec::new(); - reader.read_to_end(&mut buf)?; - Self::from_bytes_with_password(Bytes::from(buf), password) + ) -> Result + where + R: Read + Seek + Send + 'static, + { + Self::from_reader_with_password_and_options( + reader, + password, + ArchiveOpenOptions { + storage_mode: ArchiveStorageMode::Seek, + ..ArchiveOpenOptions::default() + }, + ) + } + + pub fn from_reader_with_options( + reader: R, + options: ArchiveOpenOptions, + ) -> Result + where + R: Read + Seek + Send + 'static, + { + Self::from_reader_with_password_and_options(reader, None, options) + } + + pub fn from_reader_with_password_and_options( + reader: R, + password: Option<&str>, + options: ArchiveOpenOptions, + ) -> Result + where + R: Read + Seek + Send + 'static, + { + Self::from_source_with_password(ArchiveSource::from_reader(reader)?, password, options) } /// Parse a 7z archive from in-memory bytes. @@ -161,43 +340,57 @@ impl Archive { data: Bytes, password: Option<&str>, ) -> Result { - if data.len() < 32 { + Self::from_source_with_password( + ArchiveSource::Bytes(data), + password, + ArchiveOpenOptions::default(), + ) + } + + fn from_source_with_password( + source: ArchiveSource, + password: Option<&str>, + options: ArchiveOpenOptions, + ) -> Result { + let source_len = source.len()?; + if source_len < 32 { return Err(R7zError::Parse); } - // Validate start_header_crc on raw bytes before trusting any parsed fields. - // data[8..12] = start_header_crc, data[12..32] = the covered region. - let start_crc = u32::from_le_bytes(data[8..12].try_into().map_err(|_| R7zError::Parse)?); - if crc32fast::hash(&data[12..32]) != start_crc { - return Err(R7zError::Crc); + let mut signature_bytes = [0u8; 32]; + source.read_exact_at(0, &mut signature_bytes)?; + let (_, signature) = + SignatureHeader::parse(&signature_bytes).map_err(|_| R7zError::Parse)?; + signature.validate_start_header_crc()?; + + if signature.next_header_size > options.max_metadata_bytes { + return Err(R7zError::LimitExceeded("metadata")); } - let (input, signature) = SignatureHeader::parse(&data).map_err(|_| R7zError::Parse)?; - - let offset = usize::try_from(signature.next_header_offset).map_err(|_| R7zError::Parse)?; - let (input, prop) = find_next_property_id(input, offset).map_err(|_| R7zError::Parse)?; - // Validate next_header_crc over the raw encoded/header bytes - let header_start = checked_add_usize(32, offset)?; - let header_range = checked_range(data.len(), header_start, signature.next_header_size)?; - let header_raw = &data[header_range.clone()]; - let computed_crc = crc32fast::hash(header_raw); - if computed_crc != signature.next_header_crc { + let header_start = checked_add_u64(32, signature.next_header_offset)?; + let header_range = checked_range_u64(source_len, header_start, signature.next_header_size)?; + let next_header = + Bytes::from(source.read_range_to_vec(header_range, options.max_metadata_bytes)?); + if crc32fast::hash(&next_header) != signature.next_header_crc { return Err(R7zError::Crc); } + let (prop_input, prop) = Property::parse(&next_header).map_err(|_| R7zError::Parse)?; match prop { Property::EncodedHeader => { // Parse the EncodedHeader (describes how the full header is compressed) let (_, encoded_header) = - EncodedHeader::parse(input, &data).map_err(|_| R7zError::Parse)?; + EncodedHeader::parse(prop_input, &next_header).map_err(|_| R7zError::Parse)?; // Decompress the packed header stream let pi = &encoded_header.pack_info; let ui = &encoded_header.unpack_info; - let pack_pos = usize::try_from(pi.pack_pos).map_err(|_| R7zError::Parse)?; - let data_start = checked_add_usize(32, pack_pos)?; + let data_start = checked_add_u64(32, pi.pack_pos)?; let pack_size = *pi.pack_size.first().ok_or(R7zError::Parse)?; - let data_range = checked_range(data.len(), data_start, pack_size)?; - let packed = &data[data_range]; + if pack_size > options.max_metadata_bytes { + return Err(R7zError::LimitExceeded("metadata")); + } + let data_range = checked_range_u64(source_len, data_start, pack_size)?; + let packed = source.read_range_to_vec(data_range, options.max_metadata_bytes)?; let folder = ui.parse_folder(0)?; // Find the final output stream's unpack size. @@ -226,17 +419,22 @@ impl Archive { .ok_or(R7zError::Parse)? } }; - if unpack_size > MAX_HEADER_UNPACK_BYTES { - return Err(R7zError::Parse); + if unpack_size > options.max_metadata_bytes { + return Err(R7zError::LimitExceeded("metadata")); } - let decompressed = - codec::decompress_folder_with_password(&folder, packed, unpack_size, password)?; + let decompressed = codec::decompress_folder_with_password_and_sizes( + &folder, + &packed, + unpack_size, + &ui.unpack_sizes, + password, + )?; let decompressed = Bytes::from(decompressed); let (_, header) = Header::parse(&decompressed).map_err(|_| R7zError::Parse)?; Ok(Archive { - data, + source, signature, encoded_header: Some(encoded_header), header, @@ -245,11 +443,10 @@ impl Archive { Property::Header => { // Header is stored uncompressed at next_header_offset (the raw bytes // include the 0x01 tag, so we slice from header_start, not header_start+1) - let header_bytes = data.slice(header_range); - let (_, header) = Header::parse(&header_bytes).map_err(|_| R7zError::Parse)?; + let (_, header) = Header::parse(&next_header).map_err(|_| R7zError::Parse)?; Ok(Archive { - data, + source, signature, encoded_header: None, header, @@ -363,11 +560,12 @@ impl Archive { } let location = self.extraction_location(file_index)?; - let packed = &self.data[location.packed_range.clone()]; - let mut reader = codec::folder_reader( + let packed = self.source.range_reader(location.packed_range.clone())?; + let mut reader = codec::folder_reader_with_sizes_from_reader( &location.folder, - packed, + Box::new(packed), location.folder_unpack_size, + &location.coder_unpack_sizes, password, )?; @@ -439,6 +637,19 @@ impl Archive { Ok(written) } + pub fn symlink_target(&self, file_index: usize) -> Result, R7zError> { + let Some(fi) = self.files_info() else { + return Ok(None); + }; + if !fi.is_symlink(file_index) { + return Ok(None); + } + let target = self.extract_to_memory(file_index)?; + String::from_utf8(target) + .map(Some) + .map_err(|_| R7zError::Parse) + } + fn extraction_location(&self, file_index: usize) -> Result { let fi = self.header.files_info(); let streams = self.streams_info().ok_or(R7zError::Parse)?; @@ -467,13 +678,13 @@ impl Archive { let pack_offset_u64 = prior_pack_sizes.iter().try_fold(0u64, |acc, &size| { acc.checked_add(size).ok_or(R7zError::Parse) })?; - let pack_offset = usize::try_from(pack_offset_u64).map_err(|_| R7zError::Parse)?; let pack_size = *pack_info.pack_size.get(folder_idx).ok_or(R7zError::Parse)?; - let pack_pos = usize::try_from(pack_info.pack_pos).map_err(|_| R7zError::Parse)?; - let data_start = checked_add_usize(checked_add_usize(32, pack_pos)?, pack_offset)?; - let packed_range = checked_range(self.data.len(), data_start, pack_size)?; + let data_start = + checked_add_u64(checked_add_u64(32, pack_info.pack_pos)?, pack_offset_u64)?; + let packed_range = checked_range_u64(self.source.len()?, data_start, pack_size)?; let folder_unpack_size = folder_total_unpack_size(folder_idx, unpack_info, substream_info)?; + let coder_unpack_sizes = folder_coder_unpack_sizes(folder_idx, unpack_info)?; let stream_start = stream_offset_in_folder(folder_idx, stream_in_folder, substream_info, unpack_info)?; let stream_size = @@ -490,6 +701,7 @@ impl Archive { folder, packed_range, folder_unpack_size, + coder_unpack_sizes, stream_start, stream_size, folder_digest, @@ -560,8 +772,9 @@ impl Archive { struct ExtractionLocation { folder: crate::Folder, - packed_range: Range, + packed_range: Range, folder_unpack_size: u64, + coder_unpack_sizes: Vec, stream_start: usize, stream_size: usize, folder_digest: Option, @@ -582,6 +795,14 @@ fn checked_add_usize(lhs: usize, rhs: usize) -> Result { lhs.checked_add(rhs).ok_or(R7zError::Parse) } +fn checked_add_u64(lhs: u64, rhs: u64) -> Result { + lhs.checked_add(rhs).ok_or(R7zError::Parse) +} + +fn checked_sub_u64(lhs: u64, rhs: u64) -> Result { + lhs.checked_sub(rhs).ok_or(R7zError::Parse) +} + fn checked_range(total_len: usize, start: usize, len: u64) -> Result, R7zError> { let len = usize::try_from(len).map_err(|_| R7zError::Parse)?; let end = start.checked_add(len).ok_or(R7zError::Parse)?; @@ -592,6 +813,32 @@ fn checked_range(total_len: usize, start: usize, len: u64) -> Result Result, R7zError> { + let end = start.checked_add(len).ok_or(R7zError::Parse)?; + if end <= total_len { + Ok(start..end) + } else { + Err(R7zError::Parse) + } +} + +fn folder_coder_unpack_sizes( + folder_idx: usize, + unpack_info: &crate::UnpackInfo, +) -> Result, R7zError> { + let mut global_base = 0usize; + for i in 0..folder_idx { + global_base += unpack_info.parse_folder(i)?.total_out_streams(); + } + let folder = unpack_info.parse_folder(folder_idx)?; + let num = folder.total_out_streams(); + unpack_info + .unpack_sizes + .get(global_base..global_base + num) + .map(<[u64]>::to_vec) + .ok_or(R7zError::Parse) +} + fn safe_archive_path(dest: &Path, name: &str) -> Result, R7zError> { if name.is_empty() || has_windows_prefix(name) || has_parent_component(name) { return Err(R7zError::UnsafePath(name.to_string())); @@ -701,29 +948,28 @@ fn folder_total_unpack_size( .get(global_base) .copied() .ok_or(R7zError::Parse); - } else { - // Multi-coder: find the out-stream NOT bound as an output in any bind pair - // (the one that produces the final decompressed data). - for out_idx in 0..num_out { - let is_bound = folder - .bind_pairs - .iter() - .any(|&(_, bound_out)| bound_out == out_idx as u64); - if !is_bound { - return unpack_info - .unpack_sizes - .get(global_base + out_idx) - .copied() - .ok_or(R7zError::Parse); - } + } + // Multi-coder: find the out-stream NOT bound as an output in any bind pair + // (the one that produces the final decompressed data). + for out_idx in 0..num_out { + let is_bound = folder + .bind_pairs + .iter() + .any(|&(_, bound_out)| bound_out == out_idx as u64); + if !is_bound { + return unpack_info + .unpack_sizes + .get(global_base + out_idx) + .copied() + .ok_or(R7zError::Parse); } - // Fallback: last out-stream - return unpack_info - .unpack_sizes - .get(global_base + num_out - 1) - .copied() - .ok_or(R7zError::Parse); } + // Fallback: last out-stream + return unpack_info + .unpack_sizes + .get(global_base + num_out - 1) + .copied() + .ok_or(R7zError::Parse); } // Legacy fallback: try direct index diff --git a/src/bcj.rs b/src/bcj.rs index bfcdbcc..b786592 100644 --- a/src/bcj.rs +++ b/src/bcj.rs @@ -7,7 +7,7 @@ //! //! The algorithm matches p7zip / LZMA SDK `Bra86.c` exactly. -use std::io::{self, Read}; +use std::io::{self, Read, Write}; /// Test whether the most-significant byte of a 4-byte displacement indicates /// a near address (0x00 or 0xFF after biased addition). @@ -26,6 +26,7 @@ fn test86_msb(b: u8) -> bool { /// /// Returns the number of bytes that were fully processed. Trailing bytes /// (fewer than 5) are left untouched and should be prepended to the next call. +#[allow(clippy::cast_possible_truncation)] pub fn bcj_x86_convert(data: &mut [u8], ip: u32, state: &mut u32, encoding: bool) -> usize { let size = data.len(); let mut pos: usize = 0; @@ -117,6 +118,56 @@ pub(crate) struct BcjX86Reader { eof: bool, } +pub(crate) struct BcjX86Writer { + inner: W, + tail: Vec, + state: u32, + input_offset: u64, +} + +impl BcjX86Writer { + pub(crate) fn new(inner: W) -> Self { + Self { + inner, + tail: Vec::with_capacity(4), + state: 0, + input_offset: 0, + } + } + + pub(crate) fn finish(mut self) -> io::Result { + if !self.tail.is_empty() { + self.inner.write_all(&self.tail)?; + self.tail.clear(); + } + Ok(self.inner) + } +} + +impl Write for BcjX86Writer { + fn write(&mut self, buf: &[u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + + let mut data = Vec::with_capacity(self.tail.len() + buf.len()); + data.extend_from_slice(&self.tail); + data.extend_from_slice(buf); + + #[allow(clippy::cast_possible_truncation)] + let processed = bcj_x86_convert(&mut data, self.input_offset as u32, &mut self.state, true); + self.inner.write_all(&data[..processed])?; + self.tail.clear(); + self.tail.extend_from_slice(&data[processed..]); + self.input_offset = self.input_offset.wrapping_add(processed as u64); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + impl BcjX86Reader { pub(crate) fn new(inner: R) -> Self { Self { @@ -149,6 +200,7 @@ impl BcjX86Reader { data.extend_from_slice(&self.tail); data.extend_from_slice(&chunk[..n]); + #[allow(clippy::cast_possible_truncation)] let processed = bcj_x86_convert(&mut data, self.input_offset as u32, &mut self.state, false); self.pending.extend_from_slice(&data[..processed]); @@ -201,6 +253,7 @@ pub fn bcj_x86_encode(data: &mut [u8]) { } #[cfg(test)] +#[allow(clippy::pedantic)] mod tests { use super::*; @@ -381,4 +434,26 @@ mod tests { assert_eq!(actual, original); } + + #[test] + fn streaming_encode_matches_batch_for_chunk_sizes() { + let mut data = vec![0x90u8; 4096]; + for pos in (3..data.len().saturating_sub(5)).step_by(37) { + data[pos] = if pos % 2 == 0 { 0xE8 } else { 0xE9 }; + let target = (pos as u32).wrapping_mul(11); + data[pos + 1..pos + 5].copy_from_slice(&target.to_le_bytes()); + } + + let mut batch = data.clone(); + bcj_x86_encode(&mut batch); + + for chunk_size in [1, 2, 3, 4, 5, 7, 31, 1024] { + let mut writer = BcjX86Writer::new(Vec::new()); + for chunk in data.chunks(chunk_size) { + writer.write_all(chunk).unwrap(); + } + let streamed = writer.finish().unwrap(); + assert_eq!(streamed, batch, "chunk_size={chunk_size}"); + } + } } diff --git a/src/bin/build_n64.rs b/src/bin/build_n64.rs index a162901..b17b9da 100644 --- a/src/bin/build_n64.rs +++ b/src/bin/build_n64.rs @@ -6,8 +6,7 @@ /// - `root_dir` = `/mnt/emulation/n64` /// - `output` = `/tmp/n64_build.7z` /// -/// Files are piped one at a time through the LZMA2 encoder directly to the output file. -/// Neither all input data nor the full compressed archive is held in memory. +/// Files are collected into a .7z archive and written to the output path. use r7z::build_streaming; use std::fs::File; use std::io::{self, BufWriter}; diff --git a/src/builder.rs b/src/builder.rs deleted file mode 100644 index 9b42e51..0000000 --- a/src/builder.rs +++ /dev/null @@ -1,977 +0,0 @@ -use crate::{codec, parsers::sevenzip_varuint64_encode, R7zError}; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::time::SystemTime; - -/// Codec selection for [`ArchiveBuilder`]. -#[derive(Clone, Copy, Default)] -pub enum Codec { - /// Classic LZMA (id `\[0x03, 0x01, 0x01\]`). Widely supported; lzma-rs exposes - /// the 5-byte properties directly from its `LZMA_ALONE` output. - #[default] - Lzma, - /// LZMA2 (id `\[0x21\]`). Modern default in p7zip/7-Zip; slightly better - /// compression ratio and supports multi-threading on the encode side. - Lzma2, - /// LZMA2 with x86 BCJ pre-filter. Improves compression of executable code - /// by converting relative CALL/JMP addresses to absolute form before compression. - Lzma2Bcj, -} - -/// Builds a 7z archive in memory from one or more files. -/// -/// # Example -/// ```rust,no_run -/// let bytes = r7z::ArchiveBuilder::new() -/// .add_file("hello.txt", b"Hello, world!") -/// .build() -/// .unwrap(); -/// std::fs::write("out.7z", bytes).unwrap(); -/// ``` -pub struct ArchiveBuilder { - files: Vec<(String, Vec)>, - codec: Codec, -} - -impl Default for ArchiveBuilder { - fn default() -> Self { - ArchiveBuilder::new() - } -} - -impl ArchiveBuilder { - #[must_use] - pub fn new() -> Self { - ArchiveBuilder { - files: Vec::new(), - codec: Codec::default(), - } - } - - #[must_use] - pub fn add_file(mut self, name: &str, data: &[u8]) -> Self { - self.files.push((name.to_string(), data.to_vec())); - self - } - - #[must_use] - pub fn compression(mut self, codec: Codec) -> Self { - self.codec = codec; - self - } - - /// Build the 7z archive, returning the raw bytes. - /// - /// # Errors - /// - /// Returns [`R7zError::Parse`] if no files have been added, or any compression - /// error encountered while building the archive. - pub fn build(self) -> Result, R7zError> { - if self.files.is_empty() { - return Err(R7zError::Parse); - } - build_archive(&self.files, self.codec) - } -} - -// ── ArchiveWriter ──────────────────────────────────────────────────────────── - -/// Per-entry filesystem metadata embedded in the archive header. -/// -/// All fields are optional; absent fields are not encoded in the archive. -/// Pass [`EntryMeta::default`] (all `None`) to omit metadata. -#[derive(Clone, Default)] -pub struct EntryMeta { - /// Last-modified time. Stored as Windows FILETIME (100 ns ticks since 1601-01-01). - pub mtime: Option, - /// Unix file mode (`st_mode`). Stored in the high 16 bits of the `WinAttrib` field. - pub unix_mode: Option, -} - -/// Per-file metadata held inside a [`FolderMeta`] until the header is written. -struct FileMeta { - name: String, - unpack_size: u64, - crc: u32, - entry: EntryMeta, -} - -/// Convert a [`SystemTime`] to a Windows FILETIME value. -/// -/// Windows FILETIME counts 100-nanosecond intervals since 1601-01-01T00:00:00Z. -/// The Unix epoch (1970-01-01) is 11 644 473 600 seconds after the Windows epoch. -fn system_time_to_filetime(t: SystemTime) -> u64 { - const EPOCH_DIFF_SECS: u64 = 11_644_473_600; - const TICKS_PER_SEC: u64 = 10_000_000; - match t.duration_since(std::time::UNIX_EPOCH) { - Ok(d) => { - let secs = d.as_secs().saturating_add(EPOCH_DIFF_SECS); - let subsec_ticks = u64::from(d.subsec_nanos()) / 100; - secs.saturating_mul(TICKS_PER_SEC) - .saturating_add(subsec_ticks) - } - Err(_) => 0, // pre-epoch; clamp to Windows epoch - } -} - -/// Metadata for one completed compression folder. -struct FolderMeta { - /// Per-file metadata. - files: Vec, - /// Compressed byte count for this folder's pack stream. - pack_size: u64, - /// Encoded full Folder block (num_coders + coders + bind pairs + packed indices). - coder_info: Vec, - /// Number of coder output streams in this folder (1 for single coder, 2 for BCJ+LZMA2). - num_out_streams: usize, -} - -/// Active compressor state inside [`ArchiveWriter`]. -enum CompressorState { - /// LZMA2 streaming compressor writing through a [`CountWriter`]. - Lzma2(Box>>), - /// LZMA buffers the full folder in memory before compressing on seal. - Lzma { buf: Vec, out: W }, - /// BCJ + LZMA2: buffers uncompressed data, applies BCJ encode then LZMA2 on seal. - Lzma2Bcj { buf: Vec, out: W }, -} - -/// Current I/O state of the underlying writer inside [`ArchiveWriter`]. -enum WriterState { - /// No folder is open; `W` is directly accessible. - Open(W), - /// A folder is being filled; the compressor owns `W`. - Writing(Box>), - /// Transient variant used during `seal_current_folder` to take ownership. - Dead, -} - -/// Streaming 7z archive writer with multi-folder (multi-compression-unit) support. -/// -/// Drive the write imperatively: -/// 1. Create with [`ArchiveWriter::new`]. -/// 2. Call [`append`](ArchiveWriter::append) for each file. -/// 3. Optionally call [`new_folder`](ArchiveWriter::new_folder) to start a new -/// compression unit; files added after the call go into the new folder. -/// 4. Call [`finish`](ArchiveWriter::finish) to seal the archive and recover -/// the underlying writer. -/// -/// A *folder* in 7z is an independent compression unit. Solid compression -/// (all files in one folder) yields the best ratio; separate folders allow -/// random access to individual files without decompressing everything. -/// -/// # Example -/// -/// ```rust,no_run -/// use std::fs::File; -/// -/// let file = File::create("out.7z").unwrap(); -/// let mut w = r7z::ArchiveWriter::new(file).unwrap(); -/// w.append("a.txt", &mut b"hello".as_ref()).unwrap(); -/// w.new_folder().unwrap(); -/// w.append("b.txt", &mut b"world".as_ref()).unwrap(); -/// w.finish().unwrap(); -/// ``` -pub struct ArchiveWriter { - state: WriterState, - completed: Vec, - current_files: Vec, - codec: Codec, -} - -impl ArchiveWriter { - /// Create a new [`ArchiveWriter`] writing to `out`. - /// - /// Immediately reserves 32 bytes at the start of `out` for the signature - /// header, which is filled in by [`finish`](Self::finish). - /// - /// # Errors - /// - /// Returns [`R7zError`] if the initial write to `out` fails. - pub fn new(mut out: W) -> Result { - out.write_all(&[0u8; 32]).map_err(|_| R7zError::Parse)?; - Ok(ArchiveWriter { - state: WriterState::Open(out), - completed: Vec::new(), - current_files: Vec::new(), - codec: Codec::default(), - }) - } - - /// Set the compression codec for all subsequent folders. - /// - /// Must be called before the first [`append`](Self::append) to take effect - /// on the current folder; calling it mid-folder has no effect until - /// [`new_folder`](Self::new_folder) is called. - #[must_use] - pub fn compression(mut self, codec: Codec) -> Self { - self.codec = codec; - self - } - - /// Append a file to the current compression folder. - /// - /// Equivalent to `append_entry(name, reader, EntryMeta::default())`. - /// - /// # Errors - /// - /// Returns [`R7zError`] on I/O failure or compression error. - pub fn append(&mut self, name: &str, reader: impl Read) -> Result<(), R7zError> { - self.append_entry(name, reader, EntryMeta::default()) - } - - /// Append a file with filesystem metadata to the current compression folder. - /// - /// `meta.mtime` is stored as a Windows FILETIME in the `MTime` property block. - /// `meta.unix_mode` is stored in the high 16 bits of the `Attributes` block. - /// - /// The reader is streamed directly through the compressor to the underlying - /// writer. [`Codec::Lzma`] is an exception: it must buffer the entire - /// folder in memory because LZMA has no streaming encoder. - /// - /// # Errors - /// - /// Returns [`R7zError`] on I/O failure or compression error. - pub fn append_entry( - &mut self, - name: &str, - reader: impl Read, - meta: EntryMeta, - ) -> Result<(), R7zError> { - // Transition Open → Writing on first append in a folder. - if matches!(self.state, WriterState::Open(_)) { - let state = std::mem::replace(&mut self.state, WriterState::Dead); - let WriterState::Open(w) = state else { - unreachable!() - }; - self.state = WriterState::Writing(Box::new(match self.codec { - Codec::Lzma2 => CompressorState::Lzma2(Box::new(lzma_rust2::Lzma2Writer::new( - CountWriter::new(w), - lzma_rust2::Lzma2Options::default(), - ))), - Codec::Lzma => CompressorState::Lzma { - buf: Vec::new(), - out: w, - }, - Codec::Lzma2Bcj => CompressorState::Lzma2Bcj { - buf: Vec::new(), - out: w, - }, - })); - } - - let mut hr = HashRead::new(reader); - match &mut self.state { - WriterState::Writing(cs) => match cs.as_mut() { - CompressorState::Lzma2(lzma2) => { - std::io::copy(&mut hr, lzma2.as_mut()).map_err(|_| R7zError::Parse)?; - } - CompressorState::Lzma { buf, .. } => { - std::io::copy(&mut hr, buf).map_err(|_| R7zError::Parse)?; - } - CompressorState::Lzma2Bcj { buf, .. } => { - std::io::copy(&mut hr, buf).map_err(|_| R7zError::Parse)?; - } - }, - _ => return Err(R7zError::Parse), - } - - let (crc, size) = hr.finish(); - self.current_files.push(FileMeta { - name: name.to_string(), - unpack_size: size, - crc, - entry: meta, - }); - Ok(()) - } - - /// Seal the current compression folder and start a new one. - /// - /// Files appended after this call go into a separate compression unit. - /// Calling `new_folder` when no files have been appended since the last - /// folder boundary is a no-op. - /// - /// # Errors - /// - /// Returns [`R7zError`] if sealing the current folder fails. - pub fn new_folder(&mut self) -> Result<(), R7zError> { - self.seal_current_folder() - } - - /// Seal the archive, write the 7z header and signature, and return the - /// underlying writer. - /// - /// # Errors - /// - /// Returns [`R7zError::Parse`] if no files have been added, or on any I/O - /// or compression error. - pub fn finish(mut self) -> Result { - self.seal_current_folder()?; - if self.completed.is_empty() { - return Err(R7zError::Parse); - } - - let WriterState::Open(mut w) = self.state else { - return Err(R7zError::Parse); - }; - - let next_header_offset: u64 = self.completed.iter().map(|f| f.pack_size).sum(); - let header = build_header_multi_folder(&self.completed); - let next_header_size = header.len() as u64; - let next_header_crc = crc32fast::hash(&header); - - w.write_all(&header).map_err(|_| R7zError::Parse)?; - - let mut start_header = [0u8; 20]; - start_header[..8].copy_from_slice(&next_header_offset.to_le_bytes()); - start_header[8..16].copy_from_slice(&next_header_size.to_le_bytes()); - start_header[16..].copy_from_slice(&next_header_crc.to_le_bytes()); - let start_header_crc = crc32fast::hash(&start_header); - - w.seek(SeekFrom::Start(0)).map_err(|_| R7zError::Parse)?; - let mut sig = [0u8; 32]; - sig[..6].copy_from_slice(&[0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); - sig[6] = 0x00; // major version - sig[7] = 0x04; // minor version - sig[8..12].copy_from_slice(&start_header_crc.to_le_bytes()); - sig[12..20].copy_from_slice(&next_header_offset.to_le_bytes()); - sig[20..28].copy_from_slice(&next_header_size.to_le_bytes()); - sig[28..32].copy_from_slice(&next_header_crc.to_le_bytes()); - w.write_all(&sig).map_err(|_| R7zError::Parse)?; - w.flush().map_err(|_| R7zError::Parse)?; - Ok(w) - } - - /// Compress and flush the current folder to the underlying writer. - /// - /// No-op when no files have been appended since the last folder boundary. - fn seal_current_folder(&mut self) -> Result<(), R7zError> { - if self.current_files.is_empty() { - return Ok(()); - } - - let state = std::mem::replace(&mut self.state, WriterState::Dead); - let (w, pack_size, coder_info, num_out_streams) = match state { - WriterState::Writing(cs) => match *cs { - CompressorState::Lzma2(lzma2) => { - let cw = lzma2.finish().map_err(|_| R7zError::Parse)?; - let pack_size = cw.count; - (cw.inner, pack_size, encode_coder_info_lzma2(0x1c), 1) - } - CompressorState::Lzma { buf, mut out } => { - let (props, compressed) = codec::compress_lzma(&buf)?; - let pack_size = compressed.len() as u64; - out.write_all(&compressed).map_err(|_| R7zError::Parse)?; - (out, pack_size, encode_coder_info_lzma(&props), 1) - } - CompressorState::Lzma2Bcj { mut buf, mut out } => { - // Apply BCJ encode pre-filter in place - crate::bcj::bcj_x86_encode(&mut buf); - // Compress with LZMA2 - let (props_byte, compressed) = codec::compress_lzma2(&buf)?; - let pack_size = compressed.len() as u64; - out.write_all(&compressed).map_err(|_| R7zError::Parse)?; - (out, pack_size, encode_coder_info_bcj_lzma2(props_byte), 2) - } - }, - WriterState::Open(_) | WriterState::Dead => return Err(R7zError::Parse), - }; - - self.state = WriterState::Open(w); - self.completed.push(FolderMeta { - files: std::mem::take(&mut self.current_files), - pack_size, - coder_info, - num_out_streams, - }); - Ok(()) - } -} - -fn build_archive(files: &[(String, Vec)], codec: Codec) -> Result, R7zError> { - // Solid compression: concatenate all file data into one stream. - let mut all_data: Vec = Vec::new(); - for (_, data) in files { - all_data.extend_from_slice(data); - } - - let (coder_flags_and_id_and_props, compressed, num_out_streams) = match codec { - Codec::Lzma => { - let (props, compressed) = codec::compress_lzma(&all_data)?; - (encode_coder_info_lzma(&props), compressed, 1usize) - } - Codec::Lzma2 => { - let (props_byte, compressed) = codec::compress_lzma2(&all_data)?; - (encode_coder_info_lzma2(props_byte), compressed, 1) - } - Codec::Lzma2Bcj => { - // Apply BCJ encode pre-filter - let mut filtered = all_data.clone(); - crate::bcj::bcj_x86_encode(&mut filtered); - let (props_byte, compressed) = codec::compress_lzma2(&filtered)?; - (encode_coder_info_bcj_lzma2(props_byte), compressed, 2) - } - }; - - let pack_size = compressed.len() as u64; - let folder_unpack_size = all_data.len() as u64; - - let header = build_header( - files, - &coder_flags_and_id_and_props, - pack_size, - folder_unpack_size, - num_out_streams, - ); - - // Layout: [32-byte SignatureHeader][compressed data][header] - let mut archive: Vec = vec![0u8; 32]; - archive.extend_from_slice(&compressed); - let next_header_offset = compressed.len() as u64; - archive.extend_from_slice(&header); - - // Compute and fill in the signature. - let next_header_size = header.len() as u64; - let next_header_crc = crc32fast::hash(&header); - - let mut start_header = [0u8; 20]; - start_header[..8].copy_from_slice(&next_header_offset.to_le_bytes()); - start_header[8..16].copy_from_slice(&next_header_size.to_le_bytes()); - start_header[16..].copy_from_slice(&next_header_crc.to_le_bytes()); - let start_header_crc = crc32fast::hash(&start_header); - - archive[..6].copy_from_slice(&[0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); - archive[6] = 0x00; // major version - archive[7] = 0x04; // minor version - archive[8..12].copy_from_slice(&start_header_crc.to_le_bytes()); - archive[12..20].copy_from_slice(&next_header_offset.to_le_bytes()); - archive[20..28].copy_from_slice(&next_header_size.to_le_bytes()); - archive[28..32].copy_from_slice(&next_header_crc.to_le_bytes()); - - Ok(archive) -} - -/// Counts bytes written through it. -struct CountWriter { - inner: W, - count: u64, -} - -impl CountWriter { - fn new(inner: W) -> Self { - Self { inner, count: 0 } - } -} - -impl Write for CountWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let n = self.inner.write(buf)?; - self.count += n as u64; - Ok(n) - } - fn flush(&mut self) -> std::io::Result<()> { - self.inner.flush() - } -} - -/// Hashes and counts bytes as they are read. -struct HashRead { - inner: R, - hasher: crc32fast::Hasher, - count: u64, -} - -impl HashRead { - fn new(inner: R) -> Self { - Self { - inner, - hasher: crc32fast::Hasher::new(), - count: 0, - } - } - - fn finish(self) -> (u32, u64) { - (self.hasher.finalize(), self.count) - } -} - -impl Read for HashRead { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - let n = self.inner.read(buf)?; - if n > 0 { - self.hasher.update(&buf[..n]); - self.count += n as u64; - } - Ok(n) - } -} - -/// Build a solid LZMA2 7z archive from an iterator of `(name, reader)` pairs, writing -/// directly into `out` (a file or any `Write + Seek`). -/// -/// Neither all input data nor all compressed output is held in memory simultaneously — -/// each file is piped from `reader` through the compressor into `out` as it goes. -/// -/// # Errors -/// -/// Returns [`R7zError`] on I/O or compression failure. -pub fn build_streaming(entries: I, mut out: W) -> Result<(), R7zError> -where - W: Write + Seek, - I: IntoIterator, - R: Read, -{ - // Reserve 32 bytes for the signature header (filled in at the end via seek). - out.write_all(&[0u8; 32]).map_err(|_| R7zError::Parse)?; - - // Wrap the output so we can count how many compressed bytes are written. - let count_writer = CountWriter::new(&mut out); - let mut lzma2 = lzma_rust2::Lzma2Writer::new(count_writer, lzma_rust2::Lzma2Options::default()); - - let mut file_meta: Vec<(String, u64, u32)> = Vec::new(); // (name, unpack_size, crc) - - for (name, reader) in entries { - let mut hr = HashRead::new(reader); - std::io::copy(&mut hr, &mut lzma2).map_err(|_| R7zError::Parse)?; - let (crc, size) = hr.finish(); - file_meta.push((name, size, crc)); - } - - if file_meta.is_empty() { - return Err(R7zError::Parse); - } - - // Finish the LZMA2 stream. Block-scope releases the &mut out borrow naturally. - let pack_size = { - let cw = lzma2.finish().map_err(|_| R7zError::Parse)?; - cw.count - }; - - let folder_unpack_size: u64 = file_meta.iter().map(|(_, s, _)| s).sum(); - let props_byte = 0x1c_u8; - let coder_info = encode_coder_info_lzma2(props_byte); - - let header = build_header_from_meta(&file_meta, &coder_info, pack_size, folder_unpack_size); - - out.write_all(&header).map_err(|_| R7zError::Parse)?; - - let next_header_offset = pack_size; - let next_header_size = header.len() as u64; - let next_header_crc = crc32fast::hash(&header); - - let mut start_header = [0u8; 20]; - start_header[..8].copy_from_slice(&next_header_offset.to_le_bytes()); - start_header[8..16].copy_from_slice(&next_header_size.to_le_bytes()); - start_header[16..].copy_from_slice(&next_header_crc.to_le_bytes()); - let start_header_crc = crc32fast::hash(&start_header); - - out.seek(SeekFrom::Start(0)).map_err(|_| R7zError::Parse)?; - let mut sig = [0u8; 32]; - sig[..6].copy_from_slice(&[0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); - sig[6] = 0x00; - sig[7] = 0x04; - sig[8..12].copy_from_slice(&start_header_crc.to_le_bytes()); - sig[12..20].copy_from_slice(&next_header_offset.to_le_bytes()); - sig[20..28].copy_from_slice(&next_header_size.to_le_bytes()); - sig[28..32].copy_from_slice(&next_header_crc.to_le_bytes()); - out.write_all(&sig).map_err(|_| R7zError::Parse)?; - out.flush().map_err(|_| R7zError::Parse)?; - Ok(()) -} - -/// Variant of [`build_header`] that takes pre-computed per-file metadata instead of owned data. -fn build_header_from_meta( - file_meta: &[(String, u64, u32)], - coder_info_bytes: &[u8], - pack_size: u64, - folder_unpack_size: u64, -) -> Vec { - let mut h: Vec = Vec::new(); - - h.push(0x01); // Header tag - h.push(0x04); // MainStreamsInfo tag - - // PackInfo - h.push(0x06); - h.extend_from_slice(&sevenzip_varuint64_encode(0)); - h.extend_from_slice(&sevenzip_varuint64_encode(1)); - h.push(0x09); - h.extend_from_slice(&sevenzip_varuint64_encode(pack_size)); - h.push(0x00); - - // UnpackInfo - h.push(0x07); - h.push(0x0b); - h.extend_from_slice(&sevenzip_varuint64_encode(1)); - h.push(0x00); - h.extend_from_slice(coder_info_bytes); - h.push(0x0c); - h.extend_from_slice(&sevenzip_varuint64_encode(folder_unpack_size)); - h.push(0x00); - - // SubstreamsInfo (only needed for multi-file solid archives) - if file_meta.len() > 1 { - h.push(0x08); - h.push(0x0d); - h.extend_from_slice(&sevenzip_varuint64_encode(file_meta.len() as u64)); - h.push(0x09); - for (_, size, _) in &file_meta[..file_meta.len() - 1] { - h.extend_from_slice(&sevenzip_varuint64_encode(*size)); - } - h.push(0x0a); - h.push(0x01); // all_defined - for (_, _, crc) in file_meta { - h.extend_from_slice(&crc.to_le_bytes()); - } - h.push(0x00); - } - - h.push(0x00); // END StreamInfo - - h.push(0x05); - h.extend_from_slice(&sevenzip_varuint64_encode(file_meta.len() as u64)); - - h.push(0x11); - let name_data: Vec = { - let mut nd = Vec::new(); - for (name, _, _) in file_meta { - for unit in name.encode_utf16() { - nd.extend_from_slice(&unit.to_le_bytes()); - } - nd.push(0); - nd.push(0); - } - nd - }; - let name_block_size = 1 + name_data.len() as u64; - h.extend_from_slice(&sevenzip_varuint64_encode(name_block_size)); - h.push(0x00); - h.extend_from_slice(&name_data); - - h.push(0x00); // END FilesInfo - h.push(0x00); // END Header - h -} - -/// Build a 7z `Header` block for an archive with N compression folders. -/// -/// Handles both single-folder and multi-folder archives. All files across all -/// folders are listed sequentially in `FilesInfo`. -fn build_header_multi_folder(folders: &[FolderMeta]) -> Vec { - let mut h: Vec = Vec::new(); - let num_folders = folders.len(); - let total_files: usize = folders.iter().map(|f| f.files.len()).sum(); - - h.push(0x01); // Header - h.push(0x04); // MainStreamsInfo - - // PackInfo (0x06): one pack stream per folder - h.push(0x06); - h.extend_from_slice(&sevenzip_varuint64_encode(0)); // pack_pos = 0 - h.extend_from_slice(&sevenzip_varuint64_encode(num_folders as u64)); - h.push(0x09); // Size - for folder in folders { - h.extend_from_slice(&sevenzip_varuint64_encode(folder.pack_size)); - } - h.push(0x00); // END PackInfo - - // UnpackInfo (0x07): one folder block per compression unit - h.push(0x07); - h.push(0x0b); // Folder - h.extend_from_slice(&sevenzip_varuint64_encode(num_folders as u64)); - h.push(0x00); // external = 0 - for folder in folders { - // coder_info contains the full folder block: num_coders + coders + bind pairs - h.extend_from_slice(&folder.coder_info); - } - h.push(0x0c); // CodersUnPackSize: one entry per coder out-stream - for folder in folders { - let unpack: u64 = folder.files.iter().map(|f| f.unpack_size).sum(); - // For multi-coder folders (e.g. BCJ+LZMA2), write one size per out-stream. - // BCJ doesn't change the data size, so all out-stream sizes are the same. - for _ in 0..folder.num_out_streams { - h.extend_from_slice(&sevenzip_varuint64_encode(unpack)); - } - } - h.push(0x00); // END UnpackInfo - - // SubstreamsInfo — needed when total_files > 1 to carry per-file sizes/CRCs - if total_files > 1 { - h.push(0x08); // SubstreamsInfo - - // NumUnPackStream: one varint per folder - h.push(0x0d); - for folder in folders { - h.extend_from_slice(&sevenzip_varuint64_encode(folder.files.len() as u64)); - } - - // Size: for each folder with > 1 file, (n-1) substream sizes; last is implicit - if folders.iter().any(|f| f.files.len() > 1) { - h.push(0x09); - for folder in folders { - let n = folder.files.len(); - for file in &folder.files[..n.saturating_sub(1)] { - h.extend_from_slice(&sevenzip_varuint64_encode(file.unpack_size)); - } - } - } - - // CRC: one per file across all folders - h.push(0x0a); - h.push(0x01); // all_defined = 1 - for folder in folders { - for file in &folder.files { - h.extend_from_slice(&file.crc.to_le_bytes()); - } - } - h.push(0x00); // END SubstreamsInfo - } - - h.push(0x00); // END StreamInfo - - // FilesInfo - h.push(0x05); - h.extend_from_slice(&sevenzip_varuint64_encode(total_files as u64)); - - // Names property (0x11) - h.push(0x11); - let name_data: Vec = { - let mut nd = Vec::new(); - for folder in folders { - for file in &folder.files { - for unit in file.name.encode_utf16() { - nd.extend_from_slice(&unit.to_le_bytes()); - } - nd.push(0); - nd.push(0); // UTF-16LE null terminator - } - } - nd - }; - let name_block_size = 1 + name_data.len() as u64; - h.extend_from_slice(&sevenzip_varuint64_encode(name_block_size)); - h.push(0x00); // external = 0 - h.extend_from_slice(&name_data); - - // MTime property (0x14): written when any entry has a timestamp. - // Uses all_defined=1; entries without a timestamp get FILETIME 0 (Windows epoch). - let any_mtime = folders - .iter() - .flat_map(|f| &f.files) - .any(|f| f.entry.mtime.is_some()); - if any_mtime { - h.push(0x14); // MTime tag - let block_size = 2 + 8 * total_files as u64; // all_defined + external + n×8 - h.extend_from_slice(&sevenzip_varuint64_encode(block_size)); - h.push(0x01); // all_defined = 1 - h.push(0x00); // external = 0 (data is inline) - for folder in folders { - for file in &folder.files { - let ft = file.entry.mtime.map(system_time_to_filetime).unwrap_or(0); - h.extend_from_slice(&ft.to_le_bytes()); - } - } - } - - // Attributes property (0x15): written when any entry has a Unix mode. - // High 16 bits = st_mode; low 16 bits = Windows attribs (0x20 = archive bit). - let any_attrs = folders - .iter() - .flat_map(|f| &f.files) - .any(|f| f.entry.unix_mode.is_some()); - if any_attrs { - h.push(0x15); // Attributes tag - let block_size = 2 + 4 * total_files as u64; // all_defined + external + n×4 - h.extend_from_slice(&sevenzip_varuint64_encode(block_size)); - h.push(0x01); // all_defined = 1 - h.push(0x00); // external = 0 (data is inline) - for folder in folders { - for file in &folder.files { - let attrs = file.entry.unix_mode.map_or(0x20_u32, |m| (m << 16) | 0x20); - h.extend_from_slice(&attrs.to_le_bytes()); - } - } - } - - h.push(0x00); // END FilesInfo - h.push(0x00); // END Header - h -} - -/// Encode a `CoderInfo` block for LZMA (`codec_id` = \[0x03,0x01,0x01\], 5-byte properties). -fn encode_coder_info_lzma(props: &[u8]) -> Vec { - let mut bytes = Vec::new(); - // num_coders = 1 - bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); - // flags byte: id_size=3 (bits 0-3), is_complex=0 (bit 4), has_attrs=1 (bit 5) → 0x23 - bytes.extend_from_slice(&[0x23u8, 0x03, 0x01, 0x01]); - bytes.extend_from_slice(&sevenzip_varuint64_encode(5)); // props_size - bytes.extend_from_slice(props); - // 0 bind pairs (single coder: total_out_streams = 1) - // packed_indices omitted (num_packed = 1, implicit) - bytes -} - -/// Encode a `CoderInfo` block for LZMA2 (`codec_id` = \[0x21\], 1-byte properties). -/// -/// The properties byte encodes the dictionary size hint: -/// `dict_size = 1 << (props_byte / 2 + 11)` for even values. -/// We claim 32 MB (prop = 0x1c) which covers lzma-rs's default dictionary. -fn encode_coder_info_lzma2(props_byte: u8) -> Vec { - let mut bytes = Vec::new(); - // num_coders = 1 - bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); - // flags byte: id_size=1 (bits 0-3), is_complex=0 (bit 4), has_attrs=1 (bit 5) → 0x21 - bytes.push(0x21); - bytes.push(0x21); // codec_id = [0x21] - bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); // props_size - bytes.push(props_byte); - // 0 bind pairs (single coder) - bytes -} - -/// Encode a full Folder block for BCJ x86 + LZMA2: 2 coders + 1 bind pair. -/// -/// Layout matches p7zip: coder\[0\] = LZMA2, coder\[1\] = BCJ x86. -/// Bind pair: `(in_stream=1, out_stream=0)` — BCJ input comes from LZMA2 output. -/// -/// The *packed* data stream enters coder 0 (LZMA2), whose output feeds coder 1 (BCJ). -fn encode_coder_info_bcj_lzma2(props_byte: u8) -> Vec { - let mut bytes = Vec::new(); - - // num_coders = 2 - bytes.extend_from_slice(&sevenzip_varuint64_encode(2)); - - // Coder 0: LZMA2 (has properties) - // flags: id_size=1, is_complex=0, has_attrs=1 → 0x21 - bytes.push(0x21); - bytes.push(0x21); // codec_id = [0x21] - bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); // props_size=1 - bytes.push(props_byte); - - // Coder 1: BCJ x86 (no properties) - // flags: id_size=4, is_complex=0, has_attrs=0 → 0x04 - bytes.push(0x04); - bytes.extend_from_slice(&[0x03, 0x03, 0x01, 0x03]); // codec_id - - // Bind pairs: num_out_total = 2, so num_bind_pairs = 1 - // bind_pair: (in_index=1, out_index=0) - // BCJ (coder 1, in_stream 1) reads from LZMA2 (coder 0, out_stream 0) - bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); // in_index - bytes.extend_from_slice(&sevenzip_varuint64_encode(0)); // out_index - - // num_packed_streams = num_in_total - num_bind_pairs = 2 - 1 = 1 → implicit, not written - - bytes -} - -/// Serialize the uncompressed 7z Header block. -/// -/// Layout: -/// ```text -/// 0x01 Header -/// 0x04 MainStreamsInfo -/// PackInfo UnpackInfo [SubstreamsInfo] -/// 0x00 END StreamInfo -/// 0x05 FilesInfo -/// ... -/// 0x00 END FilesInfo -/// 0x00 END Header -/// ``` -fn build_header( - files: &[(String, Vec)], - coder_info_bytes: &[u8], - pack_size: u64, - folder_unpack_size: u64, - num_out_streams: usize, -) -> Vec { - let mut h: Vec = Vec::new(); - - h.push(0x01); // Header tag - - // MainStreamsInfo - h.push(0x04); // MainStreamsInfo tag (consumed by Header loop; StreamInfo::parse starts after) - - // PackInfo (0x06): pack_pos=0, num_streams=1, one size - h.push(0x06); - h.extend_from_slice(&sevenzip_varuint64_encode(0)); // pack_pos - h.extend_from_slice(&sevenzip_varuint64_encode(1)); // num_pack_streams - h.push(0x09); // Size tag - h.extend_from_slice(&sevenzip_varuint64_encode(pack_size)); - h.push(0x00); // END PackInfo - - // UnpackInfo (0x07) - h.push(0x07); - h.push(0x0b); // Folder tag - h.extend_from_slice(&sevenzip_varuint64_encode(1)); // num_folders = 1 - h.push(0x00); // external = 0 - - // Folder: num_coders + CoderInfo (fully encoded in coder_info_bytes) - h.extend_from_slice(coder_info_bytes); - // bind_pairs and packed_indices are included in coder_info_bytes - - h.push(0x0c); // CodersUnPackSize tag - for _ in 0..num_out_streams { - h.extend_from_slice(&sevenzip_varuint64_encode(folder_unpack_size)); - } - h.push(0x00); // END UnpackInfo - - // SubstreamsInfo — only needed for solid multi-file archives - if files.len() > 1 { - h.push(0x08); // SubstreamsInfo tag (also re-read by SubstreamInfo::parse) - - // NumUnPackStream: one entry per folder - h.push(0x0d); - h.extend_from_slice(&sevenzip_varuint64_encode(files.len() as u64)); - - // Size: n-1 explicit sizes (last stream size is implicit) - h.push(0x09); - for (_, data) in &files[..files.len() - 1] { - h.extend_from_slice(&sevenzip_varuint64_encode(data.len() as u64)); - } - - // CRC: one per stream - h.push(0x0a); - h.push(0x01); // all_defined = 1 - for (_, data) in files { - let crc = crc32fast::hash(data); - h.extend_from_slice(&crc.to_le_bytes()); - } - - h.push(0x00); // END SubstreamsInfo - } - - h.push(0x00); // END StreamInfo - - // FilesInfo — the 0x05 tag is read by both the Header loop (for dispatch) and - // FilesInfo::parse (for validation), both from the same byte position. - h.push(0x05); // FilesInfo tag - h.extend_from_slice(&sevenzip_varuint64_encode(files.len() as u64)); - - // Name property - h.push(0x11); // Name tag - let name_data = encode_utf16le_names(files); - let name_block_size = 1 + name_data.len() as u64; // +1 for the external byte - h.extend_from_slice(&sevenzip_varuint64_encode(name_block_size)); - h.push(0x00); // external = 0 - h.extend_from_slice(&name_data); - - h.push(0x00); // END FilesInfo - - h.push(0x00); // END Header - - h -} - -/// Encode file names as concatenated null-terminated UTF-16LE strings. -fn encode_utf16le_names(files: &[(String, Vec)]) -> Vec { - let mut out = Vec::new(); - for (name, _) in files { - for unit in name.encode_utf16() { - out.extend_from_slice(&unit.to_le_bytes()); - } - out.push(0); - out.push(0); // UTF-16LE null terminator - } - out -} diff --git a/src/codec.rs b/src/codec.rs index c5726f3..c92a60f 100644 --- a/src/codec.rs +++ b/src/codec.rs @@ -48,6 +48,7 @@ pub const CODEC_AES_256_SHA_256: &[u8] = &[0x06, 0xF1, 0x07, 0x01]; /// The properties byte encodes the maximum dictionary size needed for decompression. /// We advertise 32 MB (0x1c), which matches the default preset dictionary. /// p7zip uses this only for memory estimation — the LZMA2 stream is self-describing. +#[allow(dead_code)] pub fn compress_lzma2(data: &[u8]) -> Result<(u8, Vec), R7zError> { let buf = Vec::new(); let mut writer = Lzma2Writer::new(buf, lzma_rust2::Lzma2Options::default()); @@ -72,12 +73,10 @@ fn lzma2_dict_size(props: Option<&[u8]>) -> Result { } let p = props[0]; - if p > 40 { - Err(R7zError::Decompression) - } else if p == 40 { - Ok(u32::MAX) - } else { - Ok((2u32 | (u32::from(p) & 1)) << ((u32::from(p) >> 1) + 11)) + match p.cmp(&40) { + std::cmp::Ordering::Greater => Err(R7zError::Decompression), + std::cmp::Ordering::Equal => Ok(u32::MAX), + std::cmp::Ordering::Less => Ok((2u32 | (u32::from(p) & 1)) << ((u32::from(p) >> 1) + 11)), } } @@ -123,7 +122,23 @@ pub fn decompress_folder_with_password( unpack_size: u64, password: Option<&str>, ) -> Result, R7zError> { - let mut reader = folder_reader(folder, packed_data, unpack_size, password)?; + decompress_folder_with_password_and_sizes(folder, packed_data, unpack_size, &[], password) +} + +pub fn decompress_folder_with_password_and_sizes( + folder: &Folder, + packed_data: &[u8], + unpack_size: u64, + coder_unpack_sizes: &[u64], + password: Option<&str>, +) -> Result, R7zError> { + let mut reader = folder_reader_with_sizes( + folder, + packed_data, + unpack_size, + coder_unpack_sizes, + password, + )?; let mut data = Vec::with_capacity(usize::try_from(unpack_size).unwrap_or(0)); reader .read_to_end(&mut data) @@ -131,14 +146,31 @@ pub fn decompress_folder_with_password( Ok(data) } -pub(crate) fn folder_reader<'a>( +pub(crate) fn folder_reader_with_sizes<'a>( folder: &Folder, packed_data: &'a [u8], unpack_size: u64, + coder_unpack_sizes: &[u64], + password: Option<&str>, +) -> Result, R7zError> { + folder_reader_with_sizes_from_reader( + folder, + Box::new(Cursor::new(packed_data)), + unpack_size, + coder_unpack_sizes, + password, + ) +} + +pub(crate) fn folder_reader_with_sizes_from_reader<'a>( + folder: &Folder, + input: Box, + unpack_size: u64, + coder_unpack_sizes: &[u64], password: Option<&str>, ) -> Result, R7zError> { let order = coder_execution_order(folder)?; - let mut reader: Box = Box::new(Cursor::new(packed_data)); + let mut reader = input; // Multi-coder chain: resolve bind-pair ordering so that each coder's // output feeds the next one's input. @@ -151,8 +183,10 @@ pub(crate) fn folder_reader<'a>( // packed stream (starts first) and following the bind pairs. for (i, &coder_idx) in order.iter().enumerate() { let coder = &folder.coders[coder_idx]; - // For chained coders we don't know intermediate sizes; use 0 to signal "unknown". - let size = if i == order.len() - 1 { unpack_size } else { 0 }; + let size = coder_unpack_sizes + .get(coder_idx) + .copied() + .unwrap_or(if i == order.len() - 1 { unpack_size } else { 0 }); reader = coder_reader(coder, reader, size, password)?; } Ok(reader) @@ -196,7 +230,10 @@ fn coder_reader<'a>( let key = crate::aes::derive_key(pwd, &props.salt, props.num_cycles_power)?; let mut encrypted = Vec::new(); read_to_end_bounded(&mut input, &mut encrypted, MAX_BUFFERED_AES_BYTES)?; - let decrypted = crate::aes::decrypt_aes256_cbc(&encrypted, &key, &props.iv)?; + let mut decrypted = crate::aes::decrypt_aes256_cbc(&encrypted, &key, &props.iv)?; + if unpack_size > 0 { + truncate_to(&mut decrypted, unpack_size)?; + } return Ok(Box::new(Cursor::new(decrypted))); } @@ -222,6 +259,15 @@ fn read_to_end_bounded( } } +fn truncate_to(data: &mut Vec, size: u64) -> Result<(), R7zError> { + let size = usize::try_from(size).map_err(|_| R7zError::Parse)?; + if data.len() < size { + return Err(R7zError::Decompression); + } + data.truncate(size); + Ok(()) +} + /// Determine the order in which coders should be executed for decompression. /// /// The packed (compressed) stream enters one coder first, its output feeds the diff --git a/src/error.rs b/src/error.rs index 21233a7..08c4846 100644 --- a/src/error.rs +++ b/src/error.rs @@ -42,4 +42,12 @@ pub enum R7zError { /// The requested entry is a directory or anti-item, not a regular file. #[error("entry is a directory or anti-item")] Directory, + + /// A writer option is outside the range supported by the 7z format. + #[error("invalid archive option: {0}")] + InvalidOptions(&'static str), + + /// A configured safety limit was exceeded. + #[error("{0} limit exceeded")] + LimitExceeded(&'static str), } diff --git a/src/files_info.rs b/src/files_info.rs index 8a60ead..788920e 100644 --- a/src/files_info.rs +++ b/src/files_info.rs @@ -9,8 +9,14 @@ pub struct FilesInfo { pub num_files: u64, /// Raw UTF-16LE null-terminated name block (empty = no Name property present). name_data: Bytes, + /// Creation timestamps as Windows FILETIME values (100ns intervals since 1601-01-01). + pub ctimes: Vec>, + /// Last-access timestamps as Windows FILETIME values (100ns intervals since 1601-01-01). + pub atimes: Vec>, /// Last-modified timestamps as Windows FILETIME values (100ns intervals since 1601-01-01). pub mtimes: Vec>, + /// Per-entry start positions. + pub start_positions: Vec>, /// Windows file attributes per entry. pub attributes: Vec>, /// Raw bitmap of empty-stream flags (empty = all false). Bit `i` = entry `i` has no data stream. @@ -23,6 +29,15 @@ pub struct FilesInfo { empty_stream_ordinals: Vec>, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EntryType { + File, + Directory, + EmptyFile, + Anti, + Symlink, +} + impl FilesInfo { /// Decode the name of entry `i` on demand (UTF-16LE, null-terminated). pub fn name(&self, i: usize) -> Option { @@ -86,7 +101,7 @@ impl FilesInfo { /// Returns `true` if entry `i` is a directory. pub fn is_directory(&self, i: usize) -> bool { - self.is_empty_stream(i) && !self.is_empty_file(i) + self.is_empty_stream(i) && !self.is_empty_file(i) && !self.is_anti(i) } /// Returns `true` if entry `i` is an anti-item. @@ -97,6 +112,28 @@ impl FilesInfo { bitmap_is_set(&self.anti_items, empty_idx) } + pub fn is_symlink(&self, i: usize) -> bool { + self.attributes + .get(i) + .copied() + .flatten() + .is_some_and(|attrs| ((attrs >> 16) & 0o170_000) == 0o120_000) + } + + pub fn entry_type(&self, i: usize) -> EntryType { + if self.is_anti(i) { + EntryType::Anti + } else if self.is_symlink(i) { + EntryType::Symlink + } else if self.is_directory(i) { + EntryType::Directory + } else if self.is_empty_file(i) { + EntryType::EmptyFile + } else { + EntryType::File + } + } + fn empty_stream_ordinal(&self, i: usize) -> Option { self.empty_stream_ordinals.get(i).copied().flatten() } @@ -133,7 +170,10 @@ impl FilesInfo { // Lazy init: no allocations until the relevant property is seen. let mut name_data = Bytes::new(); + let mut ctimes: Vec> = Vec::new(); + let mut atimes: Vec> = Vec::new(); let mut mtimes: Vec> = Vec::new(); + let mut start_positions: Vec> = Vec::new(); let mut attributes: Vec> = Vec::new(); let mut empty_streams = Bytes::new(); let mut empty_files = Bytes::new(); @@ -164,7 +204,7 @@ impl FilesInfo { name_data = backing.slice_ref(&block[1..]); input = i; } - Property::MTime => { + Property::CTime | Property::ATime | Property::MTime | Property::StartPos => { let (i, size) = sevenzip_varuint64_decode(input)?; let sz = usize::try_from(size).map_err(|_| { nom::Err::Error(nom::error::Error::new( @@ -173,37 +213,13 @@ impl FilesInfo { )) })?; let (i, block) = take(sz)(i)?; - if block.is_empty() { - return Err(nom::Err::Error(nom::error::Error::new( - input, - nom::error::ErrorKind::Eof, - ))); - } - let all_defined = block[0]; - // Layout: all_defined [bitmap if !all_defined] external [data...] - // external byte position: 1 (all_defined) or 1+nb (all_defined + bitmap) - let (data_start, num_bytes) = if all_defined != 0 { - (2usize, 0usize) // skip all_defined + external - } else { - let nb = n.div_ceil(8); - (2 + nb, nb) // skip all_defined + bitmap + external - }; - let bitmap = if all_defined != 0 { - &[][..] - } else { - &block[1..=num_bytes] - }; - mtimes = Vec::with_capacity(n.min(block.len() / 8)); - let mut pos = data_start; - for j in 0..n { - let is_def = all_defined != 0 || bitmap_is_set(bitmap, j); - if is_def && pos + 8 <= block.len() { - let val = u64::from_le_bytes(block[pos..pos + 8].try_into().unwrap()); - mtimes.push(Some(val)); - pos += 8; - } else { - mtimes.push(None); - } + let values = parse_defined_u64_property(input, block, n)?; + match tag { + Property::CTime => ctimes = values, + Property::ATime => atimes = values, + Property::MTime => mtimes = values, + Property::StartPos => start_positions = values, + _ => unreachable!(), } input = i; } @@ -216,37 +232,7 @@ impl FilesInfo { )) })?; let (i, block) = take(sz)(i)?; - if block.is_empty() { - return Err(nom::Err::Error(nom::error::Error::new( - input, - nom::error::ErrorKind::Eof, - ))); - } - let all_defined = block[0]; - // Layout: all_defined [bitmap if !all_defined] external [data...] - let (data_start, num_bytes) = if all_defined != 0 { - (2usize, 0usize) // skip all_defined + external - } else { - let nb = n.div_ceil(8); - (2 + nb, nb) // skip all_defined + bitmap + external - }; - let bitmap = if all_defined != 0 { - &[][..] - } else { - &block[1..=num_bytes] - }; - attributes = Vec::with_capacity(n.min(block.len() / 4)); - let mut pos = data_start; - for j in 0..n { - let is_def = all_defined != 0 || bitmap_is_set(bitmap, j); - if is_def && pos + 4 <= block.len() { - let val = u32::from_le_bytes(block[pos..pos + 4].try_into().unwrap()); - attributes.push(Some(val)); - pos += 4; - } else { - attributes.push(None); - } - } + attributes = parse_defined_u32_property(input, block, n)?; input = i; } Property::EmptyStream => { @@ -306,7 +292,10 @@ impl FilesInfo { FilesInfo { num_files, name_data, + ctimes, + atimes, mtimes, + start_positions, attributes, empty_streams, empty_files, @@ -332,6 +321,99 @@ fn empty_stream_ordinals(empty_streams: &[u8], num_files: usize) -> Vec = nom::Err>; +type DefinedPropertyLayout<'b> = (u8, &'b [u8], usize); + +fn defined_property_layout<'a, 'b>( + error_input: &'a [u8], + block: &'b [u8], + num_values: usize, +) -> Result, ParseError<'a>> { + if block.is_empty() { + return Err(nom::Err::Error(nom::error::Error::new( + error_input, + nom::error::ErrorKind::Eof, + ))); + } + + let all_defined = block[0]; + if all_defined != 0 { + if block.len() < 2 { + return Err(nom::Err::Error(nom::error::Error::new( + error_input, + nom::error::ErrorKind::Eof, + ))); + } + return Ok((all_defined, &[], 2)); + } + + let bitmap_len = num_values.div_ceil(8); + let data_start = 2 + bitmap_len; + if block.len() < data_start { + return Err(nom::Err::Error(nom::error::Error::new( + error_input, + nom::error::ErrorKind::Eof, + ))); + } + + let bitmap_end = 1 + bitmap_len; + Ok((all_defined, &block[1..bitmap_end], data_start)) +} + +fn parse_defined_u64_property<'a>( + error_input: &'a [u8], + block: &[u8], + num_values: usize, +) -> Result>, ParseError<'a>> { + let (all_defined, bitmap, mut pos) = defined_property_layout(error_input, block, num_values)?; + let mut values = Vec::with_capacity(num_values); + for index in 0..num_values { + let is_defined = all_defined != 0 || bitmap_is_set(bitmap, index); + if is_defined { + if pos + 8 > block.len() { + return Err(nom::Err::Error(nom::error::Error::new( + error_input, + nom::error::ErrorKind::Eof, + ))); + } + values.push(Some(u64::from_le_bytes( + block[pos..pos + 8].try_into().unwrap(), + ))); + pos += 8; + } else { + values.push(None); + } + } + Ok(values) +} + +fn parse_defined_u32_property<'a>( + error_input: &'a [u8], + block: &[u8], + num_values: usize, +) -> Result>, ParseError<'a>> { + let (all_defined, bitmap, mut pos) = defined_property_layout(error_input, block, num_values)?; + let mut values = Vec::with_capacity(num_values); + for index in 0..num_values { + let is_defined = all_defined != 0 || bitmap_is_set(bitmap, index); + if is_defined { + if pos + 4 > block.len() { + return Err(nom::Err::Error(nom::error::Error::new( + error_input, + nom::error::ErrorKind::Eof, + ))); + } + values.push(Some(u32::from_le_bytes( + block[pos..pos + 4].try_into().unwrap(), + ))); + pos += 4; + } else { + values.push(None); + } + } + Ok(values) +} + /// Walk a `FilesInfo` block without allocating. Returns `num_files`. /// /// Every sub-property is size-prefixed, so we simply verify the tag, read @@ -429,7 +511,10 @@ mod tests { let fi = FilesInfo { num_files: 4, name_data: Bytes::new(), + ctimes: Vec::new(), + atimes: Vec::new(), mtimes: Vec::new(), + start_positions: Vec::new(), attributes: Vec::new(), empty_streams: Bytes::from_static(&[0b1110_0000]), empty_files: Bytes::from_static(&[0b0100_0000]), @@ -442,6 +527,7 @@ mod tests { assert!(!fi.is_directory(1)); assert!(fi.is_empty_file(1)); assert!(fi.is_anti(2)); + assert!(!fi.is_directory(2)); assert!(!fi.is_empty_stream(9)); assert!(!fi.is_empty_file(9)); assert!(!fi.is_directory(9)); diff --git a/src/lib.rs b/src/lib.rs index 5dfacc9..d9b6f2b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,22 @@ //! let bytes = archive.extract_to_memory_with_password(0, Some("pass")).unwrap(); //! ``` //! +//! `Archive::open` is file-backed by default. Generic reader input must be +//! seekable because 7z stores stream data and authoritative metadata in +//! different file regions: +//! +//! ```compile_fail +//! struct NetworkStream; +//! +//! impl std::io::Read for NetworkStream { +//! fn read(&mut self, _buf: &mut [u8]) -> std::io::Result { +//! Ok(0) +//! } +//! } +//! +//! let _archive = r7z::Archive::from_reader(NetworkStream).unwrap(); +//! ``` +//! //! `Archive::extract_all` rejects unsafe paths such as absolute names, parent //! directory traversal, and Windows-prefixed paths. Directory entries and zero-byte //! files are handled distinctly. @@ -33,8 +49,8 @@ //! std::fs::write("out.7z", bytes).unwrap(); //! ``` //! -//! For streaming or multi-folder writes, use [`ArchiveWriter`]. [`EntryMeta`] can -//! store optional modification times and Unix mode bits. [`Codec::Lzma2Bcj`] applies +//! For file-backed or multi-folder writes, use [`ArchiveWriter`]. [`EntryMeta`] can +//! store optional timestamps and attributes. [`Codec::Lzma2Bcj`] applies //! the x86 BCJ filter before LZMA2 compression for executable-like payloads. extern crate num; @@ -44,7 +60,6 @@ extern crate num_derive; mod aes; mod archive; pub mod bcj; -mod builder; mod codec; mod coder_info; mod error; @@ -55,22 +70,28 @@ mod pack_info; mod parsers; mod property; mod stream_info; +mod write; -pub use archive::{Archive, ArchiveMetadata}; -pub use builder::{build_streaming, ArchiveBuilder, ArchiveWriter, Codec, EntryMeta}; +pub use archive::{Archive, ArchiveMetadata, ArchiveOpenOptions, ArchiveStorageMode}; pub use codec::{ decompress_folder, decompress_folder_with_password, CODEC_AES_256_SHA_256, CODEC_BCJ_X86, CODEC_COPY, CODEC_LZMA, CODEC_LZMA2, }; pub use coder_info::CoderInfo; pub use error::R7zError; -pub use files_info::FilesInfo; +pub use files_info::{EntryType, FilesInfo}; pub use folder::Folder; pub use headers::{EncodedHeader, Header, SignatureHeader}; pub use pack_info::{PackInfo, UnpackInfo}; pub use parsers::*; pub use property::{find_next_property_id, Property}; pub use stream_info::{StreamInfo, SubstreamInfo}; +pub use write::{ + build_streaming, build_streaming_to_writer, build_streaming_volumes, + build_streaming_with_options, ArchiveBuilder, ArchiveEntry, ArchiveOptions, ArchiveWriter, + Codec, CompressionLevel, CompressionOptions, EncryptionOptions, EntryKind, EntryMeta, + HeaderMode, SolidMode, SpoolMode, StreamingOptions, VolumeOptions, +}; // Re-export nom's IResult for convenience in integration tests pub use nom::IResult; diff --git a/src/write/encode.rs b/src/write/encode.rs new file mode 100644 index 0000000..250213d --- /dev/null +++ b/src/write/encode.rs @@ -0,0 +1,449 @@ +use super::header::{ + build_encoded_header_descriptor, build_header, encode_coder_info_aes_then, + encode_coder_info_bcj_lzma2, encode_coder_info_copy, encode_coder_info_lzma, + encode_coder_info_lzma2, CoderSpec, +}; +use super::model::{ + ArchiveOptions, Codec, CompletedFolder, CompressionLevel, CompressionOptions, + EncryptionOptions, HeaderMode, SolidMode, WriteEntry, +}; +use crate::{aes, bcj, codec, R7zError}; +use lzma_rust2::{Lzma2Options, Lzma2Writer, LzmaOptions, LzmaWriter}; +use std::collections::BTreeMap; +use std::io::{Seek, SeekFrom, Write}; + +type PayloadEncoding = (Vec, Vec, Vec, Vec); +type HeaderEncoding = (Vec, Vec, Vec); + +pub(crate) fn build_archive( + entries: &[WriteEntry], + options: &ArchiveOptions, +) -> Result, R7zError> { + if entries.is_empty() { + return Err(R7zError::Parse); + } + validate_archive_options(options)?; + + let mut packed_data = Vec::new(); + let mut folders = Vec::new(); + let mut by_folder: BTreeMap> = BTreeMap::new(); + for (idx, entry) in entries.iter().enumerate() { + if entry.has_stream { + by_folder.entry(entry.folder_id).or_default().push(idx); + } + } + + for file_indices in by_folder.into_values() { + let (folder, pack) = encode_folder(entries, file_indices, options)?; + packed_data.extend_from_slice(&pack); + folders.push(folder); + } + + let raw_header = build_header(entries, &folders); + let should_encode = match options.header_mode { + HeaderMode::Plain => false, + HeaderMode::Encoded => true, + HeaderMode::P7zipDefault => { + entries.len() > 1 + || options.encryption.is_some() + || options + .encryption + .as_ref() + .is_some_and(|enc| enc.encrypt_header) + } + }; + + let (next_header, next_header_offset) = if should_encode { + let (pack, coder_info, coder_unpack_sizes) = + encode_header_stream(&raw_header, options.encryption.as_ref())?; + let pack_pos = packed_data.len() as u64; + packed_data.extend_from_slice(&pack); + let descriptor = build_encoded_header_descriptor( + pack_pos, + pack.len() as u64, + &coder_info, + &coder_unpack_sizes, + ); + (descriptor, packed_data.len() as u64) + } else { + (raw_header, packed_data.len() as u64) + }; + + let mut archive = vec![0u8; 32]; + archive.extend_from_slice(&packed_data); + archive.extend_from_slice(&next_header); + write_signature(&mut archive, next_header_offset, &next_header); + Ok(archive) +} + +pub(crate) fn validate_archive_options(options: &ArchiveOptions) -> Result<(), R7zError> { + validate_compression_options(options)?; + let Some(enc) = &options.encryption else { + return Ok(()); + }; + if enc.num_cycles_power > aes::MAX_AES_NUM_CYCLES_POWER { + return Err(R7zError::InvalidOptions( + "AES num_cycles_power must be <= 24", + )); + } + if enc.salt_len > 16 { + return Err(R7zError::InvalidOptions("AES salt_len must be <= 16")); + } + if enc.iv_len > 16 { + return Err(R7zError::InvalidOptions("AES iv_len must be <= 16")); + } + if enc.encrypt_header && options.header_mode == HeaderMode::Plain { + return Err(R7zError::InvalidOptions( + "encrypt_header requires encoded headers", + )); + } + Ok(()) +} + +fn validate_compression_options(options: &ArchiveOptions) -> Result<(), R7zError> { + if options.streaming.buffer_size == 0 { + return Err(R7zError::InvalidOptions( + "streaming buffer_size must be greater than zero", + )); + } + if options.codec == Codec::Copy + && (options.compression.dictionary_size.is_some() + || options.compression.fast_bytes.is_some() + || options.compression.lzma2_chunk_size.is_some()) + { + return Err(R7zError::InvalidOptions( + "Copy codec does not support compression tuning", + )); + } + if let Some(dict) = options.compression.dictionary_size { + if dict < 4096 { + return Err(R7zError::InvalidOptions( + "dictionary_size must be at least 4096 bytes", + )); + } + } + if let Some(fast_bytes) = options.compression.fast_bytes { + if !(8..=273).contains(&fast_bytes) { + return Err(R7zError::InvalidOptions("fast_bytes must be in 8..=273")); + } + } + if let Some(chunk_size) = options.compression.lzma2_chunk_size { + let dict = lzma_options(&options.compression).dict_size; + if chunk_size.get() < u64::from(dict) { + return Err(R7zError::InvalidOptions( + "lzma2_chunk_size must be at least dictionary_size", + )); + } + } + if let SolidMode::Limit { + max_files: None, + max_bytes: None, + } = &options.compression.solid + { + return Err(R7zError::InvalidOptions( + "solid limit requires max_files or max_bytes", + )); + } + Ok(()) +} + +pub(crate) fn finish_streamed_archive( + mut out: W, + entries: &[WriteEntry], + folders: &[CompletedFolder], + options: &ArchiveOptions, +) -> Result { + if entries.is_empty() { + return Err(R7zError::Parse); + } + validate_archive_options(options)?; + + let packed_size = folders.iter().try_fold(0u64, |acc, folder| { + acc.checked_add(folder.pack_size).ok_or(R7zError::Parse) + })?; + let raw_header = build_header(entries, folders); + let should_encode = match options.header_mode { + HeaderMode::Plain => false, + HeaderMode::Encoded => true, + HeaderMode::P7zipDefault => entries.len() > 1, + }; + + let (next_header, next_header_offset) = if should_encode { + let (pack, coder_info, coder_unpack_sizes) = encode_header_stream(&raw_header, None)?; + out.seek(SeekFrom::Start(32 + packed_size))?; + out.write_all(&pack)?; + let descriptor = build_encoded_header_descriptor( + packed_size, + pack.len() as u64, + &coder_info, + &coder_unpack_sizes, + ); + (descriptor, packed_size + pack.len() as u64) + } else { + (raw_header, packed_size) + }; + + out.seek(SeekFrom::Start(32 + next_header_offset))?; + out.write_all(&next_header)?; + let signature = signature_bytes(next_header_offset, &next_header); + out.seek(SeekFrom::Start(0))?; + out.write_all(&signature)?; + out.flush()?; + Ok(out) +} + +fn encode_folder( + entries: &[WriteEntry], + file_indices: Vec, + options: &ArchiveOptions, +) -> Result<(CompletedFolder, Vec), R7zError> { + let mut data = Vec::new(); + let mut file_sizes = Vec::new(); + let mut file_crcs = Vec::new(); + for &idx in &file_indices { + let bytes = entries[idx].data.as_ref().ok_or(R7zError::Parse)?; + file_sizes.push(bytes.len() as u64); + file_crcs.push(crc32fast::hash(bytes)); + data.extend_from_slice(bytes); + } + + let (mut pack, mut coder_info, mut coder_unpack_sizes, specs) = + encode_payload_with_options(&data, options.codec, &options.compression)?; + + if let Some(enc) = &options.encryption { + let aes = make_aes_material(enc)?; + let before_padding = pack.len() as u64; + pack = aes::encrypt_aes256_cbc_zero_pad(&pack, &aes.key, &aes.iv)?; + coder_info = encode_coder_info_aes_then(&specs, &aes.props); + let mut sizes = vec![before_padding]; + sizes.append(&mut coder_unpack_sizes); + coder_unpack_sizes = sizes; + } + + Ok(( + CompletedFolder { + file_indices, + pack_size: pack.len() as u64, + coder_info, + coder_unpack_sizes, + file_sizes, + file_crcs, + }, + pack, + )) +} + +fn encode_payload_with_options( + data: &[u8], + method: Codec, + compression: &CompressionOptions, +) -> Result { + match method { + Codec::Copy => Ok(( + data.to_vec(), + encode_coder_info_copy(), + vec![data.len() as u64], + vec![CoderSpec::Copy], + )), + Codec::Lzma => { + let (props, compressed) = compress_lzma(data, compression)?; + Ok(( + compressed, + encode_coder_info_lzma(&props), + vec![data.len() as u64], + vec![CoderSpec::Lzma(props)], + )) + } + Codec::Lzma2 => { + let (prop, compressed) = compress_lzma2(data, compression)?; + Ok(( + compressed, + encode_coder_info_lzma2(prop), + vec![data.len() as u64], + vec![CoderSpec::Lzma2(prop)], + )) + } + Codec::Lzma2Bcj => { + let mut filtered = data.to_vec(); + bcj::bcj_x86_encode(&mut filtered); + let (prop, compressed) = compress_lzma2(&filtered, compression)?; + Ok(( + compressed, + encode_coder_info_bcj_lzma2(prop), + vec![data.len() as u64, data.len() as u64], + vec![CoderSpec::Lzma2(prop), CoderSpec::Bcj], + )) + } + } +} + +fn encode_header_stream( + raw_header: &[u8], + encryption: Option<&EncryptionOptions>, +) -> Result { + let (props, compressed) = codec::compress_lzma(raw_header)?; + let coder_info = encode_coder_info_lzma(&props); + let sizes = vec![raw_header.len() as u64]; + + let Some(enc) = encryption.filter(|enc| enc.encrypt_header) else { + return Ok((compressed, coder_info, sizes)); + }; + + let aes = make_aes_material(enc)?; + let before_padding = compressed.len() as u64; + let encrypted = aes::encrypt_aes256_cbc_zero_pad(&compressed, &aes.key, &aes.iv)?; + let coder_info = encode_coder_info_aes_then(&[CoderSpec::Lzma(props)], &aes.props); + Ok(( + encrypted, + coder_info, + vec![before_padding, raw_header.len() as u64], + )) +} + +pub(crate) fn lzma_options(compression: &CompressionOptions) -> LzmaOptions { + let mut options = LzmaOptions::with_preset(compression_level_preset(compression.level)); + if let Some(dict_size) = compression.dictionary_size { + options.dict_size = dict_size; + } + if let Some(fast_bytes) = compression.fast_bytes { + options.nice_len = fast_bytes; + } + options +} + +pub(crate) fn lzma2_options(compression: &CompressionOptions) -> Lzma2Options { + let mut options = Lzma2Options { + lzma_options: lzma_options(compression), + chunk_size: None, + }; + options.set_chunk_size(compression.lzma2_chunk_size); + options +} + +pub(crate) fn lzma2_property_byte(compression: &CompressionOptions) -> Result { + encode_lzma2_dict_size(lzma_options(compression).dict_size) +} + +fn compression_level_preset(level: CompressionLevel) -> u32 { + match level { + CompressionLevel::Store => 0, + CompressionLevel::Fastest => 1, + CompressionLevel::Fast => 3, + CompressionLevel::Normal => 6, + CompressionLevel::Maximum => 7, + CompressionLevel::Ultra => 9, + } +} + +fn compress_lzma( + data: &[u8], + compression: &CompressionOptions, +) -> Result<(Vec, Vec), R7zError> { + let options = lzma_options(compression); + let dict_size = options.dict_size; + let mut writer = LzmaWriter::new_no_header(Vec::new(), &options, false) + .map_err(|_| R7zError::Decompression)?; + writer + .write_all(data) + .map_err(|_| R7zError::Decompression)?; + let props_byte = writer.props(); + let compressed = writer.finish().map_err(|_| R7zError::Decompression)?; + let mut props = Vec::with_capacity(5); + props.push(props_byte); + props.extend_from_slice(&dict_size.to_le_bytes()); + Ok((props, compressed)) +} + +fn compress_lzma2( + data: &[u8], + compression: &CompressionOptions, +) -> Result<(u8, Vec), R7zError> { + let options = lzma2_options(compression); + let prop = encode_lzma2_dict_size(options.lzma_options.dict_size)?; + let mut writer = Lzma2Writer::new(Vec::new(), options); + writer + .write_all(data) + .map_err(|_| R7zError::Decompression)?; + let compressed = writer.finish().map_err(|_| R7zError::Decompression)?; + Ok((prop, compressed)) +} + +fn encode_lzma2_dict_size(dict_size: u32) -> Result { + if dict_size < 4096 { + return Err(R7zError::InvalidOptions( + "dictionary_size must be at least 4096 bytes", + )); + } + if dict_size == u32::MAX { + return Ok(40); + } + for prop in 0u8..40 { + let base = 2u32 | (u32::from(prop) & 1); + let size = base + .checked_shl((u32::from(prop) >> 1) + 11) + .ok_or(R7zError::InvalidOptions("dictionary_size is too large"))?; + if size >= dict_size { + return Ok(prop); + } + } + Err(R7zError::InvalidOptions("dictionary_size is too large")) +} + +struct AesMaterial { + key: [u8; 32], + iv: [u8; 16], + props: Vec, +} + +fn make_aes_material(options: &EncryptionOptions) -> Result { + if options.num_cycles_power > aes::MAX_AES_NUM_CYCLES_POWER { + return Err(R7zError::InvalidOptions( + "AES num_cycles_power must be <= 24", + )); + } + if options.salt_len > 16 { + return Err(R7zError::InvalidOptions("AES salt_len must be <= 16")); + } + if options.iv_len > 16 { + return Err(R7zError::InvalidOptions("AES iv_len must be <= 16")); + } + + let mut salt = vec![0u8; usize::from(options.salt_len)]; + let mut iv_bytes = vec![0u8; usize::from(options.iv_len)]; + if !salt.is_empty() { + getrandom::fill(&mut salt).map_err(|_| R7zError::Parse)?; + } + if !iv_bytes.is_empty() { + getrandom::fill(&mut iv_bytes).map_err(|_| R7zError::Parse)?; + } + let mut iv = [0u8; 16]; + let iv_copy_len = iv_bytes.len().min(16); + iv[..iv_copy_len].copy_from_slice(&iv_bytes[..iv_copy_len]); + let key = aes::derive_key(&options.password, &salt, options.num_cycles_power)?; + let props = aes::encode_aes_properties(options.num_cycles_power, &salt, &iv_bytes); + Ok(AesMaterial { key, iv, props }) +} + +fn write_signature(archive: &mut [u8], next_header_offset: u64, next_header: &[u8]) { + archive[..32].copy_from_slice(&signature_bytes(next_header_offset, next_header)); +} + +fn signature_bytes(next_header_offset: u64, next_header: &[u8]) -> [u8; 32] { + let next_header_size = next_header.len() as u64; + let next_header_crc = crc32fast::hash(next_header); + let mut start_header = [0u8; 20]; + start_header[..8].copy_from_slice(&next_header_offset.to_le_bytes()); + start_header[8..16].copy_from_slice(&next_header_size.to_le_bytes()); + start_header[16..].copy_from_slice(&next_header_crc.to_le_bytes()); + let start_header_crc = crc32fast::hash(&start_header); + + let mut signature = [0u8; 32]; + signature[..6].copy_from_slice(&[0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]); + signature[6] = 0x00; + signature[7] = 0x04; + signature[8..12].copy_from_slice(&start_header_crc.to_le_bytes()); + signature[12..20].copy_from_slice(&next_header_offset.to_le_bytes()); + signature[20..28].copy_from_slice(&next_header_size.to_le_bytes()); + signature[28..32].copy_from_slice(&next_header_crc.to_le_bytes()); + signature +} diff --git a/src/write/header.rs b/src/write/header.rs new file mode 100644 index 0000000..bd29182 --- /dev/null +++ b/src/write/header.rs @@ -0,0 +1,479 @@ +use super::model::{CompletedFolder, EntryKind, WriteEntry}; +use crate::parsers::sevenzip_varuint64_encode; +use std::time::SystemTime; + +pub(crate) fn encode_coder_info_copy() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); + bytes.push(0x01); + bytes.push(0x00); + bytes +} + +pub(crate) fn encode_coder_info_lzma(props: &[u8]) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); + bytes.extend_from_slice(&[0x23, 0x03, 0x01, 0x01]); + bytes.extend_from_slice(&sevenzip_varuint64_encode(5)); + bytes.extend_from_slice(props); + bytes +} + +pub(crate) fn encode_coder_info_lzma2(props_byte: u8) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); + bytes.push(0x21); + bytes.push(0x21); + bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); + bytes.push(props_byte); + bytes +} + +pub(crate) fn encode_coder_info_bcj_lzma2(props_byte: u8) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&sevenzip_varuint64_encode(2)); + bytes.push(0x21); + bytes.push(0x21); + bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); + bytes.push(props_byte); + bytes.push(0x04); + bytes.extend_from_slice(&[0x03, 0x03, 0x01, 0x03]); + bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); + bytes.extend_from_slice(&sevenzip_varuint64_encode(0)); + bytes +} + +pub(crate) fn encode_coder_info_aes_then(inner: &[CoderSpec], aes_props: &[u8]) -> Vec { + let mut bytes = Vec::new(); + let num_coders = 1 + inner.len() as u64; + bytes.extend_from_slice(&sevenzip_varuint64_encode(num_coders)); + + bytes.push(0x24); + bytes.extend_from_slice(&[0x06, 0xF1, 0x07, 0x01]); + bytes.extend_from_slice(&sevenzip_varuint64_encode(aes_props.len() as u64)); + bytes.extend_from_slice(aes_props); + + for spec in inner { + match spec { + CoderSpec::Copy => { + bytes.push(0x01); + bytes.push(0x00); + } + CoderSpec::Lzma(props) => { + bytes.extend_from_slice(&[0x23, 0x03, 0x01, 0x01]); + bytes.extend_from_slice(&sevenzip_varuint64_encode(5)); + bytes.extend_from_slice(props); + } + CoderSpec::Lzma2(prop) => { + bytes.push(0x21); + bytes.push(0x21); + bytes.extend_from_slice(&sevenzip_varuint64_encode(1)); + bytes.push(*prop); + } + CoderSpec::Bcj => { + bytes.push(0x04); + bytes.extend_from_slice(&[0x03, 0x03, 0x01, 0x03]); + } + } + } + + for out_idx in 0..inner.len() { + bytes.extend_from_slice(&sevenzip_varuint64_encode((out_idx + 1) as u64)); + bytes.extend_from_slice(&sevenzip_varuint64_encode(out_idx as u64)); + } + bytes +} + +#[derive(Clone)] +pub(crate) enum CoderSpec { + Copy, + Lzma(Vec), + Lzma2(u8), + Bcj, +} + +pub(crate) fn build_header(entries: &[WriteEntry], folders: &[CompletedFolder]) -> Vec { + let mut h = Vec::new(); + h.push(0x01); + + if !folders.is_empty() { + h.push(0x04); + write_pack_info(&mut h, folders); + write_unpack_info(&mut h, folders); + write_substreams_info(&mut h, folders); + h.push(0x00); + } + + write_files_info(&mut h, entries); + h.push(0x00); + h +} + +pub(crate) fn build_encoded_header_descriptor( + pack_pos: u64, + pack_size: u64, + coder_info: &[u8], + coder_unpack_sizes: &[u64], +) -> Vec { + let mut h = Vec::new(); + h.push(0x17); + h.push(0x06); + h.extend_from_slice(&sevenzip_varuint64_encode(pack_pos)); + h.extend_from_slice(&sevenzip_varuint64_encode(1)); + h.push(0x09); + h.extend_from_slice(&sevenzip_varuint64_encode(pack_size)); + h.push(0x00); + + h.push(0x07); + h.push(0x0b); + h.extend_from_slice(&sevenzip_varuint64_encode(1)); + h.push(0x00); + h.extend_from_slice(coder_info); + h.push(0x0c); + for &size in coder_unpack_sizes { + h.extend_from_slice(&sevenzip_varuint64_encode(size)); + } + h.push(0x00); + h.push(0x00); + h +} + +fn write_pack_info(h: &mut Vec, folders: &[CompletedFolder]) { + h.push(0x06); + h.extend_from_slice(&sevenzip_varuint64_encode(0)); + h.extend_from_slice(&sevenzip_varuint64_encode(folders.len() as u64)); + h.push(0x09); + for folder in folders { + h.extend_from_slice(&sevenzip_varuint64_encode(folder.pack_size)); + } + h.push(0x00); +} + +fn write_unpack_info(h: &mut Vec, folders: &[CompletedFolder]) { + h.push(0x07); + h.push(0x0b); + h.extend_from_slice(&sevenzip_varuint64_encode(folders.len() as u64)); + h.push(0x00); + for folder in folders { + h.extend_from_slice(&folder.coder_info); + } + h.push(0x0c); + for folder in folders { + for &size in &folder.coder_unpack_sizes { + h.extend_from_slice(&sevenzip_varuint64_encode(size)); + } + } + h.push(0x00); +} + +fn write_substreams_info(h: &mut Vec, folders: &[CompletedFolder]) { + h.push(0x08); + if folders.iter().any(|f| f.file_indices.len() != 1) { + h.push(0x0d); + for folder in folders { + h.extend_from_slice(&sevenzip_varuint64_encode(folder.file_indices.len() as u64)); + } + } + if folders.iter().any(|f| f.file_indices.len() > 1) { + h.push(0x09); + for folder in folders { + for &size in &folder.file_sizes[..folder.file_sizes.len().saturating_sub(1)] { + h.extend_from_slice(&sevenzip_varuint64_encode(size)); + } + } + } + h.push(0x0a); + h.push(0x01); + for folder in folders { + for &crc in &folder.file_crcs { + h.extend_from_slice(&crc.to_le_bytes()); + } + } + h.push(0x00); +} + +fn write_files_info(h: &mut Vec, entries: &[WriteEntry]) { + h.push(0x05); + h.extend_from_slice(&sevenzip_varuint64_encode(entries.len() as u64)); + write_empty_properties(h, entries); + write_names(h, entries); + write_time_property(h, 0x12, entries, |e| e.meta.ctime); + write_time_property(h, 0x13, entries, |e| e.meta.atime); + write_time_property(h, 0x14, entries, |e| e.meta.mtime); + write_u64_property(h, 0x18, entries, |e| e.meta.start_pos); + write_u32_property(h, 0x15, entries, |e| e.meta.attributes); + h.push(0x00); +} + +fn write_empty_properties(h: &mut Vec, entries: &[WriteEntry]) { + let empty: Vec = entries.iter().map(|entry| !entry.has_stream).collect(); + if !empty.iter().any(|&v| v) { + return; + } + h.push(0x0e); + let empty_bytes = bools_to_bytes(&empty); + h.extend_from_slice(&sevenzip_varuint64_encode(empty_bytes.len() as u64)); + h.extend_from_slice(&empty_bytes); + + let mut empty_files = Vec::new(); + let mut anti = Vec::new(); + for entry in entries.iter().filter(|entry| !entry.has_stream) { + empty_files.push(entry.kind == EntryKind::File); + anti.push(entry.kind == EntryKind::Anti); + } + if empty_files.iter().any(|&v| v) { + h.push(0x0f); + let bytes = bools_to_bytes(&empty_files); + h.extend_from_slice(&sevenzip_varuint64_encode(bytes.len() as u64)); + h.extend_from_slice(&bytes); + } + if anti.iter().any(|&v| v) { + h.push(0x10); + let bytes = bools_to_bytes(&anti); + h.extend_from_slice(&sevenzip_varuint64_encode(bytes.len() as u64)); + h.extend_from_slice(&bytes); + } +} + +fn write_names(h: &mut Vec, entries: &[WriteEntry]) { + h.push(0x11); + let mut name_data = Vec::new(); + for entry in entries { + for unit in entry.name.encode_utf16() { + name_data.extend_from_slice(&unit.to_le_bytes()); + } + name_data.extend_from_slice(&[0, 0]); + } + h.extend_from_slice(&sevenzip_varuint64_encode(1 + name_data.len() as u64)); + h.push(0x00); + h.extend_from_slice(&name_data); +} + +fn write_time_property( + h: &mut Vec, + tag: u8, + entries: &[WriteEntry], + value: impl Fn(&WriteEntry) -> Option, +) { + write_u64_property(h, tag, entries, |entry| { + value(entry).map(system_time_to_filetime) + }); +} + +fn write_u64_property( + h: &mut Vec, + tag: u8, + entries: &[WriteEntry], + value: impl Fn(&WriteEntry) -> Option, +) { + let values: Vec> = entries.iter().map(value).collect(); + if !values.iter().any(Option::is_some) { + return; + } + let all_defined = values.iter().all(Option::is_some); + let bitmap = (!all_defined) + .then(|| bools_to_bytes(&values.iter().map(Option::is_some).collect::>())); + let data_len = values.iter().filter(|v| v.is_some()).count() * 8; + let size = 1 + bitmap.as_ref().map_or(0, Vec::len) + 1 + data_len; + h.push(tag); + h.extend_from_slice(&sevenzip_varuint64_encode(size as u64)); + h.push(u8::from(all_defined)); + if let Some(bitmap) = bitmap { + h.extend_from_slice(&bitmap); + } + h.push(0x00); + for val in values.into_iter().flatten() { + h.extend_from_slice(&val.to_le_bytes()); + } +} + +fn write_u32_property( + h: &mut Vec, + tag: u8, + entries: &[WriteEntry], + value: impl Fn(&WriteEntry) -> Option, +) { + let values: Vec> = entries.iter().map(value).collect(); + if !values.iter().any(Option::is_some) { + return; + } + let all_defined = values.iter().all(Option::is_some); + let bitmap = (!all_defined) + .then(|| bools_to_bytes(&values.iter().map(Option::is_some).collect::>())); + let data_len = values.iter().filter(|v| v.is_some()).count() * 4; + let size = 1 + bitmap.as_ref().map_or(0, Vec::len) + 1 + data_len; + h.push(tag); + h.extend_from_slice(&sevenzip_varuint64_encode(size as u64)); + h.push(u8::from(all_defined)); + if let Some(bitmap) = bitmap { + h.extend_from_slice(&bitmap); + } + h.push(0x00); + for val in values.into_iter().flatten() { + h.extend_from_slice(&val.to_le_bytes()); + } +} + +fn bools_to_bytes(values: &[bool]) -> Vec { + let mut out = vec![0u8; values.len().div_ceil(8)]; + for (idx, &value) in values.iter().enumerate() { + if value { + out[idx / 8] |= 1 << (7 - (idx % 8)); + } + } + out +} + +fn system_time_to_filetime(t: SystemTime) -> u64 { + const EPOCH_DIFF_SECS: u64 = 11_644_473_600; + const TICKS_PER_SEC: u64 = 10_000_000; + match t.duration_since(std::time::UNIX_EPOCH) { + Ok(d) => { + let secs = d.as_secs().saturating_add(EPOCH_DIFF_SECS); + let subsec_ticks = u64::from(d.subsec_nanos()) / 100; + secs.saturating_mul(TICKS_PER_SEC) + .saturating_add(subsec_ticks) + } + Err(_) => 0, + } +} + +#[cfg(test)] +mod tests { + use super::build_header; + use crate::write::model::{EntryKind, EntryMeta, WriteEntry}; + use bytes::Bytes; + use std::time::{Duration, UNIX_EPOCH}; + + fn entry(name: &str, kind: EntryKind, has_stream: bool, meta: EntryMeta) -> WriteEntry { + WriteEntry { + name: name.to_string(), + kind, + meta, + has_stream, + data: has_stream.then(|| vec![0xAA]), + folder_id: 0, + } + } + + fn filetime_from_unix_secs(secs: u64) -> u64 { + (secs + 11_644_473_600) * 10_000_000 + } + + fn parse_header(header: Vec) -> crate::Header { + let backing = Bytes::from(header); + let (_, parsed) = crate::Header::parse(&backing).unwrap(); + parsed + } + + #[test] + fn files_info_writer_emits_empty_stream_empty_file_and_anti_bitmaps() { + let entries = vec![ + entry("dir", EntryKind::Directory, false, EntryMeta::default()), + entry("empty.txt", EntryKind::File, false, EntryMeta::default()), + entry("deleted.txt", EntryKind::Anti, false, EntryMeta::default()), + entry("data.txt", EntryKind::File, true, EntryMeta::default()), + ]; + + let header = parse_header(build_header(&entries, &[])); + let fi = header.files_info().unwrap(); + + assert_eq!(fi.empty_streams.as_ref(), &[0b1110_0000]); + assert_eq!(fi.empty_files.as_ref(), &[0b0100_0000]); + assert_eq!(fi.anti_items.as_ref(), &[0b0010_0000]); + assert!(fi.is_directory(0)); + assert!(fi.is_empty_file(1)); + assert!(fi.is_anti(2)); + assert!(!fi.is_empty_stream(3)); + } + + #[test] + fn files_info_writer_emits_partial_metadata_definition_bitmaps() { + let ctime_secs = 1_577_836_800; + let atime_secs = 1_609_459_200; + let mtime_secs = 1_640_995_200; + let entries = vec![ + entry( + "ctime-mtime.txt", + EntryKind::File, + true, + EntryMeta { + ctime: Some(UNIX_EPOCH + Duration::from_secs(ctime_secs)), + mtime: Some(UNIX_EPOCH + Duration::from_secs(mtime_secs)), + ..EntryMeta::default() + }, + ), + entry( + "atime-attrs.txt", + EntryKind::File, + true, + EntryMeta { + atime: Some(UNIX_EPOCH + Duration::from_secs(atime_secs)), + attributes: Some(0x20), + ..EntryMeta::default() + }, + ), + entry( + "start-pos.txt", + EntryKind::File, + true, + EntryMeta { + mtime: Some(UNIX_EPOCH + Duration::from_secs(mtime_secs + 60)), + start_pos: Some(77), + ..EntryMeta::default() + }, + ), + ]; + + let header = parse_header(build_header(&entries, &[])); + let fi = header.files_info().unwrap(); + + assert_eq!( + fi.ctimes, + vec![Some(filetime_from_unix_secs(ctime_secs)), None, None] + ); + assert_eq!( + fi.atimes, + vec![None, Some(filetime_from_unix_secs(atime_secs)), None] + ); + assert_eq!( + fi.mtimes, + vec![ + Some(filetime_from_unix_secs(mtime_secs)), + None, + Some(filetime_from_unix_secs(mtime_secs + 60)), + ] + ); + assert_eq!(fi.start_positions, vec![None, None, Some(77)]); + assert_eq!(fi.attributes, vec![None, Some(0x20), None]); + } + + #[test] + fn files_info_writer_uses_raw_win_attributes_from_meta() { + let entries = vec![ + entry( + "dir", + EntryKind::Directory, + false, + EntryMeta::directory_unix_mode(0o040_755), + ), + entry( + "file.txt", + EntryKind::File, + true, + EntryMeta::from_unix_mode(0o100_644), + ), + ]; + + let header = parse_header(build_header(&entries, &[])); + let fi = header.files_info().unwrap(); + + assert_eq!( + fi.attributes, + vec![ + Some((0o040_755 << 16) | 0x10), + Some((0o100_644 << 16) | 0x20) + ] + ); + } +} diff --git a/src/write/mod.rs b/src/write/mod.rs new file mode 100644 index 0000000..cc89d8d --- /dev/null +++ b/src/write/mod.rs @@ -0,0 +1,1307 @@ +#![allow(clippy::missing_errors_doc)] + +mod encode; +mod header; +mod model; + +use crate::{bcj::BcjX86Writer, R7zError}; +use header::{ + encode_coder_info_bcj_lzma2, encode_coder_info_copy, encode_coder_info_lzma, + encode_coder_info_lzma2, +}; +use lzma_rust2::{Lzma2Writer, LzmaWriter}; +use std::{ + fs::{File, OpenOptions}, + io::{self, Cursor, Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; + +pub use model::{ + ArchiveEntry, ArchiveOptions, Codec, CompressionLevel, CompressionOptions, EncryptionOptions, + EntryKind, EntryMeta, HeaderMode, SolidMode, SpoolMode, StreamingOptions, VolumeOptions, +}; + +use model::WriteEntry; + +struct StreamingCopyFolder { + file_indices: Vec, + pack_size: u64, + file_sizes: Vec, + file_crcs: Vec, +} + +impl StreamingCopyFolder { + fn new() -> Self { + Self { + file_indices: Vec::new(), + pack_size: 0, + file_sizes: Vec::new(), + file_crcs: Vec::new(), + } + } +} + +struct CountingWriter { + inner: W, + count: u64, +} + +impl Write for CountingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let n = self.inner.write(buf)?; + self.count = self.count.checked_add(n as u64).ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "archive stream too large") + })?; + Ok(n) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +struct StreamingLzma2Folder { + writer: Lzma2Writer>, + file_indices: Vec, + unpack_size: u64, + file_sizes: Vec, + file_crcs: Vec, +} + +struct StreamingBcjLzma2Folder { + writer: BcjX86Writer>>, + file_indices: Vec, + unpack_size: u64, + file_sizes: Vec, + file_crcs: Vec, +} + +struct StreamingLzmaFolder { + writer: LzmaWriter>, + props: Vec, + file_indices: Vec, + unpack_size: u64, + file_sizes: Vec, + file_crcs: Vec, +} + +pub struct ArchiveBuilder { + entries: Vec, + options: ArchiveOptions, +} + +impl Default for ArchiveBuilder { + fn default() -> Self { + Self::new() + } +} + +impl ArchiveBuilder { + #[must_use] + pub fn new() -> Self { + Self { + entries: Vec::new(), + options: ArchiveOptions::default(), + } + } + + #[must_use] + pub fn options(mut self, options: ArchiveOptions) -> Self { + self.options = options; + self + } + + #[must_use] + pub fn compression(mut self, codec: Codec) -> Self { + self.options.codec = codec; + self + } + + #[must_use] + pub fn add_file(mut self, name: &str, data: &[u8]) -> Self { + if data.is_empty() { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta: EntryMeta::default(), + has_stream: false, + data: None, + folder_id: 0, + }); + } else { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta: EntryMeta::default(), + has_stream: true, + data: Some(data.to_vec()), + folder_id: 0, + }); + } + self + } + + #[must_use] + pub fn add_file_entry(mut self, name: &str, data: &[u8], meta: EntryMeta) -> Self { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: !data.is_empty(), + data: (!data.is_empty()).then(|| data.to_vec()), + folder_id: 0, + }); + self + } + + #[must_use] + pub fn add_symlink(mut self, name: &str, target: &str, meta: EntryMeta) -> Self { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta: meta.with_symlink_default(), + has_stream: true, + data: Some(target.as_bytes().to_vec()), + folder_id: 0, + }); + self + } + + pub fn add_entry(mut self, entry: ArchiveEntry, data: Option<&[u8]>) -> Result { + self.entries.push(write_entry_from_archive_entry( + entry, + data.map(<[u8]>::to_vec), + 0, + )?); + Ok(self) + } + + #[must_use] + pub fn add_empty_file(mut self, name: &str, meta: EntryMeta) -> Self { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: false, + data: None, + folder_id: 0, + }); + self + } + + #[must_use] + pub fn add_directory(mut self, name: &str, meta: EntryMeta) -> Self { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::Directory, + meta, + has_stream: false, + data: None, + folder_id: 0, + }); + self + } + + #[must_use] + pub fn add_anti_item(mut self, name: &str, meta: EntryMeta) -> Self { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::Anti, + meta, + has_stream: false, + data: None, + folder_id: 0, + }); + self + } + + pub fn build(self) -> Result, R7zError> { + let entries = entries_with_solid_folders(self.entries, &self.options.compression.solid)?; + encode::build_archive(&entries, &self.options) + } +} + +pub struct ArchiveWriter { + out: Option, + entries: Vec, + options: ArchiveOptions, + current_folder: usize, + current_folder_files: u64, + current_folder_bytes: u64, + copy_current: StreamingCopyFolder, + copy_completed: Vec, + lzma2_current: Option>, + lzma2_completed: Vec, + bcj_lzma2_current: Option>, + bcj_lzma2_completed: Vec, + lzma_current: Option>, + lzma_completed: Vec, +} + +impl ArchiveWriter { + pub fn new(out: W, options: ArchiveOptions) -> Result { + encode::validate_archive_options(&options)?; + Ok(Self { + out: Some(out), + entries: Vec::new(), + options, + current_folder: 0, + current_folder_files: 0, + current_folder_bytes: 0, + copy_current: StreamingCopyFolder::new(), + copy_completed: Vec::new(), + lzma2_current: None, + lzma2_completed: Vec::new(), + bcj_lzma2_current: None, + bcj_lzma2_completed: Vec::new(), + lzma_current: None, + lzma_completed: Vec::new(), + }) + } + + pub fn new_default(out: W) -> Result { + Self::new(out, ArchiveOptions::default()) + } + + #[must_use] + pub fn compression(mut self, codec: Codec) -> Self { + self.options.codec = codec; + self + } + + pub fn append(&mut self, name: &str, reader: impl Read) -> Result<(), R7zError> { + self.append_file(name, reader, EntryMeta::default()) + } + + pub fn append_entry( + &mut self, + name: &str, + reader: impl Read, + meta: EntryMeta, + ) -> Result<(), R7zError> { + self.append_file(name, reader, meta) + } + + pub fn append_archive_entry( + &mut self, + entry: ArchiveEntry, + reader: impl Read, + ) -> Result<(), R7zError> { + let ArchiveEntry { name, kind, meta } = entry; + if kind != EntryKind::File { + return Err(R7zError::InvalidOptions( + "only file entries can have stream data", + )); + } + self.append_file(&name, reader, meta) + } + + pub fn append_empty_entry(&mut self, entry: ArchiveEntry) -> Result<(), R7zError> { + match entry.kind { + EntryKind::File => self.append_empty_file(&entry.name, entry.meta), + EntryKind::Directory => self.append_directory(&entry.name, entry.meta), + EntryKind::Anti => self.append_anti_item(&entry.name, entry.meta), + } + } + + pub fn append_file( + &mut self, + name: &str, + mut reader: impl Read, + meta: EntryMeta, + ) -> Result<(), R7zError> { + if self.should_stream_copy() { + return self.append_copy_streaming(name, reader, meta); + } + if self.should_stream_lzma2() { + return self.append_lzma2_streaming(name, reader, meta); + } + if self.should_stream_lzma() { + return self.append_lzma_streaming(name, reader, meta); + } + if self.should_stream_bcj_lzma2() { + return self.append_bcj_lzma2_streaming(name, reader, meta); + } + + let mut data = Vec::new(); + reader.read_to_end(&mut data)?; + let size = data.len() as u64; + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: !data.is_empty(), + data: (!data.is_empty()).then_some(data), + folder_id: self.current_folder, + }); + self.finish_entry_folder_accounting(size)?; + Ok(()) + } + + pub fn append_symlink( + &mut self, + name: &str, + target: &str, + meta: EntryMeta, + ) -> Result<(), R7zError> { + self.append_file(name, target.as_bytes(), meta.with_symlink_default()) + } + + pub fn append_empty_file(&mut self, name: &str, meta: EntryMeta) -> Result<(), R7zError> { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: false, + data: None, + folder_id: self.current_folder, + }); + Ok(()) + } + + pub fn append_directory(&mut self, name: &str, meta: EntryMeta) -> Result<(), R7zError> { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::Directory, + meta, + has_stream: false, + data: None, + folder_id: self.current_folder, + }); + Ok(()) + } + + pub fn append_anti_item(&mut self, name: &str, meta: EntryMeta) -> Result<(), R7zError> { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::Anti, + meta, + has_stream: false, + data: None, + folder_id: self.current_folder, + }); + Ok(()) + } + + pub fn new_folder(&mut self) -> Result<(), R7zError> { + if self.should_stream_copy() { + self.seal_copy_folder(); + } else if self.should_stream_lzma2() { + self.seal_lzma2_folder()?; + } else if self.should_stream_lzma() { + self.seal_lzma_folder()?; + } else if self.should_stream_bcj_lzma2() { + self.seal_bcj_lzma2_folder()?; + } else if self + .entries + .iter() + .any(|entry| entry.folder_id == self.current_folder && entry.has_stream) + { + self.current_folder += 1; + } + self.current_folder_files = 0; + self.current_folder_bytes = 0; + Ok(()) + } + + fn finish_entry_folder_accounting(&mut self, size: u64) -> Result<(), R7zError> { + if size == 0 { + return Ok(()); + } + self.current_folder_files = self + .current_folder_files + .checked_add(1) + .ok_or(R7zError::Parse)?; + self.current_folder_bytes = self + .current_folder_bytes + .checked_add(size) + .ok_or(R7zError::Parse)?; + match &self.options.compression.solid { + SolidMode::Solid => Ok(()), + SolidMode::NonSolid => self.new_folder(), + SolidMode::Limit { + max_files, + max_bytes, + } => { + let files_hit = max_files.is_some_and(|n| self.current_folder_files >= n.get()); + let bytes_hit = max_bytes.is_some_and(|n| self.current_folder_bytes >= n.get()); + if files_hit || bytes_hit { + self.new_folder() + } else { + Ok(()) + } + } + } + } + + pub fn finish(mut self) -> Result { + if !self.copy_completed.is_empty() || !self.copy_current.file_indices.is_empty() { + self.seal_copy_folder(); + let folders: Vec = self + .copy_completed + .into_iter() + .map(model::CompletedFolder::from) + .collect(); + return encode::finish_streamed_archive( + self.out.take().ok_or(R7zError::Parse)?, + &self.entries, + &folders, + &self.options, + ); + } + if self.lzma2_current.is_some() || !self.lzma2_completed.is_empty() { + self.seal_lzma2_folder()?; + return encode::finish_streamed_archive( + self.out.take().ok_or(R7zError::Parse)?, + &self.entries, + &self.lzma2_completed, + &self.options, + ); + } + if self.lzma_current.is_some() || !self.lzma_completed.is_empty() { + self.seal_lzma_folder()?; + return encode::finish_streamed_archive( + self.out.take().ok_or(R7zError::Parse)?, + &self.entries, + &self.lzma_completed, + &self.options, + ); + } + if self.bcj_lzma2_current.is_some() || !self.bcj_lzma2_completed.is_empty() { + self.seal_bcj_lzma2_folder()?; + return encode::finish_streamed_archive( + self.out.take().ok_or(R7zError::Parse)?, + &self.entries, + &self.bcj_lzma2_completed, + &self.options, + ); + } + + let bytes = encode::build_archive(&self.entries, &self.options)?; + let out = self.out.as_mut().ok_or(R7zError::Parse)?; + out.seek(SeekFrom::Start(0))?; + out.write_all(&bytes)?; + out.flush()?; + self.out.take().ok_or(R7zError::Parse) + } + + fn should_stream_copy(&self) -> bool { + self.options.codec == Codec::Copy && self.options.encryption.is_none() + } + + fn should_stream_lzma2(&self) -> bool { + self.options.codec == Codec::Lzma2 && self.options.encryption.is_none() + } + + fn should_stream_lzma(&self) -> bool { + self.options.codec == Codec::Lzma && self.options.encryption.is_none() + } + + fn should_stream_bcj_lzma2(&self) -> bool { + self.options.codec == Codec::Lzma2Bcj && self.options.encryption.is_none() + } + + fn append_copy_streaming( + &mut self, + name: &str, + mut reader: impl Read, + meta: EntryMeta, + ) -> Result<(), R7zError> { + let mut hasher = crc32fast::Hasher::new(); + let mut size = 0u64; + let mut buf = vec![0u8; self.options.streaming.buffer_size]; + let first = reader.read(&mut buf)?; + if first == 0 { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: false, + data: None, + folder_id: self.current_folder, + }); + return Ok(()); + } + self.ensure_copy_stream_started()?; + self.out + .as_mut() + .ok_or(R7zError::Parse)? + .write_all(&buf[..first])?; + hasher.update(&buf[..first]); + size = size.checked_add(first as u64).ok_or(R7zError::Parse)?; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + self.out + .as_mut() + .ok_or(R7zError::Parse)? + .write_all(&buf[..n])?; + hasher.update(&buf[..n]); + size = size.checked_add(n as u64).ok_or(R7zError::Parse)?; + } + + let index = self.entries.len(); + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: size > 0, + data: None, + folder_id: self.current_folder, + }); + + self.copy_current.file_indices.push(index); + self.copy_current.pack_size = self + .copy_current + .pack_size + .checked_add(size) + .ok_or(R7zError::Parse)?; + self.copy_current.file_sizes.push(size); + self.copy_current.file_crcs.push(hasher.finalize()); + + self.finish_entry_folder_accounting(size)?; + Ok(()) + } + + fn ensure_copy_stream_started(&mut self) -> Result<(), R7zError> { + if !self.copy_completed.is_empty() || !self.copy_current.file_indices.is_empty() { + return Ok(()); + } + let out = self.out.as_mut().ok_or(R7zError::Parse)?; + out.seek(SeekFrom::Start(0))?; + out.write_all(&[0u8; 32])?; + Ok(()) + } + + fn seal_copy_folder(&mut self) { + if !self.copy_current.file_indices.is_empty() { + self.copy_completed.push(std::mem::replace( + &mut self.copy_current, + StreamingCopyFolder::new(), + )); + self.current_folder += 1; + } + } + + fn append_lzma2_streaming( + &mut self, + name: &str, + mut reader: impl Read, + meta: EntryMeta, + ) -> Result<(), R7zError> { + let mut buf = vec![0u8; self.options.streaming.buffer_size]; + let first = reader.read(&mut buf)?; + if first == 0 { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: false, + data: None, + folder_id: self.current_folder, + }); + return Ok(()); + } + + self.ensure_lzma2_folder()?; + let mut hasher = crc32fast::Hasher::new(); + let mut size = 0u64; + self.write_lzma2_file_chunk(&buf[..first], &mut hasher, &mut size)?; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + self.write_lzma2_file_chunk(&buf[..n], &mut hasher, &mut size)?; + } + + let index = self.entries.len(); + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: true, + data: None, + folder_id: self.current_folder, + }); + let folder = self.lzma2_current.as_mut().ok_or(R7zError::Parse)?; + folder.file_indices.push(index); + folder.unpack_size = folder + .unpack_size + .checked_add(size) + .ok_or(R7zError::Parse)?; + folder.file_sizes.push(size); + folder.file_crcs.push(hasher.finalize()); + self.finish_entry_folder_accounting(size)?; + Ok(()) + } + + fn write_lzma2_file_chunk( + &mut self, + chunk: &[u8], + hasher: &mut crc32fast::Hasher, + size: &mut u64, + ) -> Result<(), R7zError> { + let folder = self.lzma2_current.as_mut().ok_or(R7zError::Parse)?; + folder + .writer + .write_all(chunk) + .map_err(|_| R7zError::Decompression)?; + hasher.update(chunk); + *size = size + .checked_add(chunk.len() as u64) + .ok_or(R7zError::Parse)?; + Ok(()) + } + + fn ensure_lzma2_folder(&mut self) -> Result<(), R7zError> { + if self.lzma2_current.is_some() { + return Ok(()); + } + if self.lzma2_completed.is_empty() { + let out = self.out.as_mut().ok_or(R7zError::Parse)?; + out.seek(SeekFrom::Start(0))?; + out.write_all(&[0u8; 32])?; + } + let out = self.out.take().ok_or(R7zError::Parse)?; + let writer = Lzma2Writer::new( + CountingWriter { + inner: out, + count: 0, + }, + encode::lzma2_options(&self.options.compression), + ); + self.lzma2_current = Some(StreamingLzma2Folder { + writer, + file_indices: Vec::new(), + unpack_size: 0, + file_sizes: Vec::new(), + file_crcs: Vec::new(), + }); + Ok(()) + } + + fn seal_lzma2_folder(&mut self) -> Result<(), R7zError> { + let Some(folder) = self.lzma2_current.take() else { + return Ok(()); + }; + let StreamingLzma2Folder { + writer, + file_indices, + unpack_size, + file_sizes, + file_crcs, + } = folder; + let count_writer = writer.finish().map_err(|_| R7zError::Decompression)?; + let pack_size = count_writer.count; + self.out = Some(count_writer.inner); + self.lzma2_completed.push(model::CompletedFolder { + file_indices, + pack_size, + coder_info: encode_coder_info_lzma2(encode::lzma2_property_byte( + &self.options.compression, + )?), + coder_unpack_sizes: vec![unpack_size], + file_sizes, + file_crcs, + }); + self.current_folder += 1; + Ok(()) + } + + fn append_lzma_streaming( + &mut self, + name: &str, + mut reader: impl Read, + meta: EntryMeta, + ) -> Result<(), R7zError> { + let mut buf = vec![0u8; self.options.streaming.buffer_size]; + let first = reader.read(&mut buf)?; + if first == 0 { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: false, + data: None, + folder_id: self.current_folder, + }); + return Ok(()); + } + + self.ensure_lzma_folder()?; + let mut hasher = crc32fast::Hasher::new(); + let mut size = 0u64; + self.write_lzma_file_chunk(&buf[..first], &mut hasher, &mut size)?; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + self.write_lzma_file_chunk(&buf[..n], &mut hasher, &mut size)?; + } + + let index = self.entries.len(); + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: true, + data: None, + folder_id: self.current_folder, + }); + let folder = self.lzma_current.as_mut().ok_or(R7zError::Parse)?; + folder.file_indices.push(index); + folder.unpack_size = folder + .unpack_size + .checked_add(size) + .ok_or(R7zError::Parse)?; + folder.file_sizes.push(size); + folder.file_crcs.push(hasher.finalize()); + self.finish_entry_folder_accounting(size)?; + Ok(()) + } + + fn write_lzma_file_chunk( + &mut self, + chunk: &[u8], + hasher: &mut crc32fast::Hasher, + size: &mut u64, + ) -> Result<(), R7zError> { + let folder = self.lzma_current.as_mut().ok_or(R7zError::Parse)?; + folder + .writer + .write_all(chunk) + .map_err(|_| R7zError::Decompression)?; + hasher.update(chunk); + *size = size + .checked_add(chunk.len() as u64) + .ok_or(R7zError::Parse)?; + Ok(()) + } + + fn ensure_lzma_folder(&mut self) -> Result<(), R7zError> { + if self.lzma_current.is_some() { + return Ok(()); + } + if self.lzma_completed.is_empty() { + let out = self.out.as_mut().ok_or(R7zError::Parse)?; + out.seek(SeekFrom::Start(0))?; + out.write_all(&[0u8; 32])?; + } + let out = self.out.take().ok_or(R7zError::Parse)?; + let options = encode::lzma_options(&self.options.compression); + let dict_size = options.dict_size; + let writer = LzmaWriter::new_no_header( + CountingWriter { + inner: out, + count: 0, + }, + &options, + false, + ) + .map_err(|_| R7zError::Decompression)?; + let mut props = Vec::with_capacity(5); + props.push(writer.props()); + props.extend_from_slice(&dict_size.to_le_bytes()); + self.lzma_current = Some(StreamingLzmaFolder { + writer, + props, + file_indices: Vec::new(), + unpack_size: 0, + file_sizes: Vec::new(), + file_crcs: Vec::new(), + }); + Ok(()) + } + + fn seal_lzma_folder(&mut self) -> Result<(), R7zError> { + let Some(folder) = self.lzma_current.take() else { + return Ok(()); + }; + let StreamingLzmaFolder { + writer, + props, + file_indices, + unpack_size, + file_sizes, + file_crcs, + } = folder; + let count_writer = writer.finish().map_err(|_| R7zError::Decompression)?; + let pack_size = count_writer.count; + self.out = Some(count_writer.inner); + self.lzma_completed.push(model::CompletedFolder { + file_indices, + pack_size, + coder_info: encode_coder_info_lzma(&props), + coder_unpack_sizes: vec![unpack_size], + file_sizes, + file_crcs, + }); + self.current_folder += 1; + Ok(()) + } + + fn append_bcj_lzma2_streaming( + &mut self, + name: &str, + mut reader: impl Read, + meta: EntryMeta, + ) -> Result<(), R7zError> { + let mut buf = vec![0u8; self.options.streaming.buffer_size]; + let first = reader.read(&mut buf)?; + if first == 0 { + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: false, + data: None, + folder_id: self.current_folder, + }); + return Ok(()); + } + + self.ensure_bcj_lzma2_folder()?; + let mut hasher = crc32fast::Hasher::new(); + let mut size = 0u64; + self.write_bcj_lzma2_file_chunk(&buf[..first], &mut hasher, &mut size)?; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + self.write_bcj_lzma2_file_chunk(&buf[..n], &mut hasher, &mut size)?; + } + + let index = self.entries.len(); + self.entries.push(WriteEntry { + name: name.to_string(), + kind: EntryKind::File, + meta, + has_stream: true, + data: None, + folder_id: self.current_folder, + }); + let folder = self.bcj_lzma2_current.as_mut().ok_or(R7zError::Parse)?; + folder.file_indices.push(index); + folder.unpack_size = folder + .unpack_size + .checked_add(size) + .ok_or(R7zError::Parse)?; + folder.file_sizes.push(size); + folder.file_crcs.push(hasher.finalize()); + self.finish_entry_folder_accounting(size)?; + Ok(()) + } + + fn write_bcj_lzma2_file_chunk( + &mut self, + chunk: &[u8], + hasher: &mut crc32fast::Hasher, + size: &mut u64, + ) -> Result<(), R7zError> { + let folder = self.bcj_lzma2_current.as_mut().ok_or(R7zError::Parse)?; + folder + .writer + .write_all(chunk) + .map_err(|_| R7zError::Decompression)?; + hasher.update(chunk); + *size = size + .checked_add(chunk.len() as u64) + .ok_or(R7zError::Parse)?; + Ok(()) + } + + fn ensure_bcj_lzma2_folder(&mut self) -> Result<(), R7zError> { + if self.bcj_lzma2_current.is_some() { + return Ok(()); + } + if self.bcj_lzma2_completed.is_empty() { + let out = self.out.as_mut().ok_or(R7zError::Parse)?; + out.seek(SeekFrom::Start(0))?; + out.write_all(&[0u8; 32])?; + } + let out = self.out.take().ok_or(R7zError::Parse)?; + let lzma2 = Lzma2Writer::new( + CountingWriter { + inner: out, + count: 0, + }, + encode::lzma2_options(&self.options.compression), + ); + self.bcj_lzma2_current = Some(StreamingBcjLzma2Folder { + writer: BcjX86Writer::new(lzma2), + file_indices: Vec::new(), + unpack_size: 0, + file_sizes: Vec::new(), + file_crcs: Vec::new(), + }); + Ok(()) + } + + fn seal_bcj_lzma2_folder(&mut self) -> Result<(), R7zError> { + let Some(folder) = self.bcj_lzma2_current.take() else { + return Ok(()); + }; + let StreamingBcjLzma2Folder { + writer, + file_indices, + unpack_size, + file_sizes, + file_crcs, + } = folder; + let lzma2 = writer.finish().map_err(|_| R7zError::Decompression)?; + let count_writer = lzma2.finish().map_err(|_| R7zError::Decompression)?; + let pack_size = count_writer.count; + self.out = Some(count_writer.inner); + self.bcj_lzma2_completed.push(model::CompletedFolder { + file_indices, + pack_size, + coder_info: encode_coder_info_bcj_lzma2(encode::lzma2_property_byte( + &self.options.compression, + )?), + coder_unpack_sizes: vec![unpack_size, unpack_size], + file_sizes, + file_crcs, + }); + self.current_folder += 1; + Ok(()) + } +} + +impl From for model::CompletedFolder { + fn from(folder: StreamingCopyFolder) -> Self { + Self { + file_indices: folder.file_indices, + pack_size: folder.pack_size, + coder_info: encode_coder_info_copy(), + coder_unpack_sizes: vec![folder.pack_size], + file_sizes: folder.file_sizes, + file_crcs: folder.file_crcs, + } + } +} + +fn entries_with_solid_folders( + mut entries: Vec, + solid: &SolidMode, +) -> Result, R7zError> { + let mut folder_id = 0usize; + let mut folder_files = 0u64; + let mut folder_bytes = 0u64; + + for entry in &mut entries { + if !entry.has_stream { + entry.folder_id = folder_id; + continue; + } + let size = entry + .data + .as_ref() + .map(|data| data.len() as u64) + .ok_or(R7zError::Parse)?; + + let would_exceed = match solid { + SolidMode::Solid | SolidMode::NonSolid => false, + SolidMode::Limit { + max_files, + max_bytes, + } => { + let next_files = folder_files.checked_add(1).ok_or(R7zError::Parse)?; + let next_bytes = folder_bytes.checked_add(size).ok_or(R7zError::Parse)?; + let files_hit = max_files.is_some_and(|n| folder_files > 0 && next_files > n.get()); + let bytes_hit = max_bytes.is_some_and(|n| folder_files > 0 && next_bytes > n.get()); + files_hit || bytes_hit + } + }; + if would_exceed { + folder_id = folder_id.checked_add(1).ok_or(R7zError::Parse)?; + folder_files = 0; + folder_bytes = 0; + } + + entry.folder_id = folder_id; + folder_files = folder_files.checked_add(1).ok_or(R7zError::Parse)?; + folder_bytes = folder_bytes.checked_add(size).ok_or(R7zError::Parse)?; + + if matches!(solid, SolidMode::NonSolid) { + folder_id = folder_id.checked_add(1).ok_or(R7zError::Parse)?; + folder_files = 0; + folder_bytes = 0; + } + } + + Ok(entries) +} + +fn write_entry_from_archive_entry( + entry: ArchiveEntry, + data: Option>, + folder_id: usize, +) -> Result { + let has_stream = entry.kind == EntryKind::File && data.as_ref().is_some_and(|d| !d.is_empty()); + if entry.kind != EntryKind::File && data.as_ref().is_some_and(|d| !d.is_empty()) { + return Err(R7zError::InvalidOptions( + "only file entries can have stream data", + )); + } + Ok(WriteEntry { + name: entry.name, + kind: entry.kind, + meta: entry.meta, + has_stream, + data: has_stream.then_some(data).flatten(), + folder_id, + }) +} + +pub fn build_streaming(entries: I, out: W) -> Result<(), R7zError> +where + W: Write + Seek, + I: IntoIterator, + R: Read, +{ + build_streaming_with_options(entries, out, ArchiveOptions::default()) +} + +pub fn build_streaming_with_options( + entries: I, + out: W, + options: ArchiveOptions, +) -> Result<(), R7zError> +where + W: Write + Seek, + I: IntoIterator, + R: Read, +{ + let mut writer = ArchiveWriter::new(out, options)?; + for (name, reader) in entries { + writer.append(&name, reader)?; + } + writer.finish()?; + Ok(()) +} + +pub fn build_streaming_to_writer( + entries: I, + mut out: W, + options: ArchiveOptions, +) -> Result<(), R7zError> +where + W: Write, + I: IntoIterator, + R: Read, +{ + encode::validate_archive_options(&options)?; + match options.streaming.spool.clone() { + SpoolMode::Memory => { + let mut spool = Cursor::new(Vec::new()); + build_streaming_with_options(entries, &mut spool, options)?; + out.write_all(spool.get_ref())?; + out.flush()?; + Ok(()) + } + SpoolMode::Auto { + memory_threshold, + dir, + } => { + let mut spool = AutoSpool::new(memory_threshold, dir)?; + let result = (|| { + build_streaming_with_options(entries, &mut spool, options)?; + spool.seek(SeekFrom::Start(0))?; + io::copy(&mut spool, &mut out)?; + out.flush()?; + Ok(()) + })(); + let remove_result = spool.cleanup(); + match (result, remove_result) { + (Err(err), _) => Err(err), + (Ok(()), Err(err)) => Err(err.into()), + (Ok(()), Ok(())) => Ok(()), + } + } + SpoolMode::TempFile { dir } => { + let (mut spool, path) = create_temp_spool(dir.as_deref())?; + let result = (|| { + build_streaming_with_options(entries, &mut spool, options)?; + spool.seek(SeekFrom::Start(0))?; + io::copy(&mut spool, &mut out)?; + out.flush()?; + Ok(()) + })(); + let remove_result = std::fs::remove_file(&path); + match (result, remove_result) { + (Err(err), _) => Err(err), + (Ok(()), Err(err)) => Err(err.into()), + (Ok(()), Ok(())) => Ok(()), + } + } + } +} + +pub fn build_streaming_volumes( + entries: I, + base_path: P, + archive_options: ArchiveOptions, + volume_options: VolumeOptions, +) -> Result, R7zError> +where + P: AsRef, + I: IntoIterator, + R: Read, +{ + if volume_options.sizes.is_empty() { + return Err(R7zError::InvalidOptions( + "volume options require at least one size", + )); + } + + let mut archive = Vec::new(); + build_streaming_to_writer(entries, &mut archive, archive_options)?; + + let base = base_path.as_ref(); + let mut paths = Vec::new(); + let mut offset = 0usize; + let mut volume_idx = 0usize; + while offset < archive.len() || (archive.is_empty() && volume_idx == 0) { + let size_idx = volume_idx.min(volume_options.sizes.len() - 1); + let size = usize::try_from(volume_options.sizes[size_idx].get()) + .map_err(|_| R7zError::InvalidOptions("volume size is too large"))?; + let end = offset.saturating_add(size).min(archive.len()); + let path = PathBuf::from(format!("{}.{:03}", base.display(), volume_idx + 1)); + let mut file = File::create(&path)?; + file.write_all(&archive[offset..end])?; + file.flush()?; + paths.push(path); + offset = end; + volume_idx += 1; + if size == 0 { + return Err(R7zError::InvalidOptions( + "volume size must be greater than zero", + )); + } + } + + Ok(paths) +} + +fn create_temp_spool(dir: Option<&Path>) -> Result<(File, PathBuf), R7zError> { + let dir = dir + .map(Path::to_path_buf) + .unwrap_or_else(std::env::temp_dir); + std::fs::create_dir_all(&dir)?; + for attempt in 0..100u32 { + let mut random = [0u8; 8]; + getrandom::fill(&mut random).map_err(|_| R7zError::Parse)?; + let name = format!( + "r7z-spool-{}-{attempt}-{:016x}.tmp", + std::process::id(), + u64::from_le_bytes(random) + ); + let path = dir.join(name); + match OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + { + Ok(file) => return Ok((file, path)), + Err(err) if err.kind() == io::ErrorKind::AlreadyExists => continue, + Err(err) => return Err(err.into()), + } + } + Err(R7zError::InvalidOptions("could not create temp spool file")) +} + +enum AutoSpoolInner { + Memory(Cursor>), + TempFile { file: File, path: PathBuf }, +} + +struct AutoSpool { + memory_threshold: u64, + dir: Option, + inner: AutoSpoolInner, +} + +impl AutoSpool { + fn new(memory_threshold: u64, dir: Option) -> Result { + let inner = if memory_threshold == 0 { + let (file, path) = create_temp_spool(dir.as_deref())?; + AutoSpoolInner::TempFile { file, path } + } else { + AutoSpoolInner::Memory(Cursor::new(Vec::new())) + }; + Ok(Self { + memory_threshold, + dir, + inner, + }) + } + + fn maybe_roll_to_file(&mut self, write_len: usize) -> io::Result<()> { + let AutoSpoolInner::Memory(cursor) = &mut self.inner else { + return Ok(()); + }; + + let projected_len = cursor + .position() + .saturating_add(write_len as u64) + .max(cursor.get_ref().len() as u64); + if projected_len <= self.memory_threshold { + return Ok(()); + } + + let current_pos = cursor.position(); + let (mut file, path) = create_temp_spool(self.dir.as_deref()).map_err(io::Error::other)?; + file.write_all(cursor.get_ref())?; + file.seek(SeekFrom::Start(current_pos))?; + self.inner = AutoSpoolInner::TempFile { file, path }; + Ok(()) + } + + fn cleanup(self) -> io::Result<()> { + match self.inner { + AutoSpoolInner::Memory(_) => Ok(()), + AutoSpoolInner::TempFile { path, .. } => std::fs::remove_file(path), + } + } +} + +impl Write for AutoSpool { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.maybe_roll_to_file(buf.len())?; + match &mut self.inner { + AutoSpoolInner::Memory(cursor) => cursor.write(buf), + AutoSpoolInner::TempFile { file, .. } => file.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match &mut self.inner { + AutoSpoolInner::Memory(cursor) => cursor.flush(), + AutoSpoolInner::TempFile { file, .. } => file.flush(), + } + } +} + +impl Read for AutoSpool { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match &mut self.inner { + AutoSpoolInner::Memory(cursor) => cursor.read(buf), + AutoSpoolInner::TempFile { file, .. } => file.read(buf), + } + } +} + +impl Seek for AutoSpool { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match &mut self.inner { + AutoSpoolInner::Memory(cursor) => cursor.seek(pos), + AutoSpoolInner::TempFile { file, .. } => file.seek(pos), + } + } +} diff --git a/src/write/model.rs b/src/write/model.rs new file mode 100644 index 0000000..ec3381b --- /dev/null +++ b/src/write/model.rs @@ -0,0 +1,239 @@ +use std::{num::NonZeroU64, path::PathBuf, time::SystemTime}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Codec { + Copy, + Lzma, + #[default] + Lzma2, + Lzma2Bcj, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum HeaderMode { + #[default] + P7zipDefault, + Plain, + Encoded, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ArchiveOptions { + pub codec: Codec, + pub header_mode: HeaderMode, + pub encryption: Option, + pub compression: CompressionOptions, + pub streaming: StreamingOptions, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CompressionOptions { + pub level: CompressionLevel, + pub dictionary_size: Option, + pub fast_bytes: Option, + pub solid: SolidMode, + pub lzma2_chunk_size: Option, +} + +impl Default for CompressionOptions { + fn default() -> Self { + Self { + level: CompressionLevel::Normal, + dictionary_size: None, + fast_bytes: None, + solid: SolidMode::Solid, + lzma2_chunk_size: None, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum CompressionLevel { + Store, + Fastest, + Fast, + #[default] + Normal, + Maximum, + Ultra, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum SolidMode { + #[default] + Solid, + NonSolid, + Limit { + max_files: Option, + max_bytes: Option, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StreamingOptions { + pub buffer_size: usize, + pub spool: SpoolMode, +} + +impl Default for StreamingOptions { + fn default() -> Self { + Self { + buffer_size: 8192, + spool: SpoolMode::Auto { + memory_threshold: 16 * 1024 * 1024, + dir: None, + }, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SpoolMode { + Memory, + TempFile { + dir: Option, + }, + Auto { + memory_threshold: u64, + dir: Option, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct VolumeOptions { + pub sizes: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncryptionOptions { + pub password: String, + pub encrypt_header: bool, + pub num_cycles_power: u8, + pub salt_len: u8, + pub iv_len: u8, +} + +impl EncryptionOptions { + #[must_use] + pub fn default_for_password(password: impl Into) -> Self { + Self { + password: password.into(), + encrypt_header: false, + num_cycles_power: 19, + salt_len: 0, + iv_len: 16, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EntryKind { + File, + Directory, + Anti, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct EntryMeta { + pub ctime: Option, + pub atime: Option, + pub mtime: Option, + pub attributes: Option, + pub start_pos: Option, +} + +impl EntryMeta { + #[must_use] + pub fn from_unix_mode(mode: u32) -> Self { + Self { + attributes: Some((mode << 16) | 0x20), + ..Self::default() + } + } + + #[must_use] + pub fn directory_unix_mode(mode: u32) -> Self { + Self { + attributes: Some((mode << 16) | 0x10), + ..Self::default() + } + } + + #[must_use] + pub fn archive_file() -> Self { + Self { + attributes: Some(0x20), + ..Self::default() + } + } + + #[must_use] + pub fn symlink() -> Self { + Self { + attributes: Some((0o120_777 << 16) | 0x20), + ..Self::default() + } + } + + pub(crate) fn with_symlink_default(mut self) -> Self { + if self.attributes.is_none() { + self.attributes = Some((0o120_777 << 16) | 0x20); + } + self + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArchiveEntry { + pub name: String, + pub kind: EntryKind, + pub meta: EntryMeta, +} + +impl ArchiveEntry { + #[must_use] + pub fn file(name: impl Into, meta: EntryMeta) -> Self { + Self { + name: name.into(), + kind: EntryKind::File, + meta, + } + } + + #[must_use] + pub fn directory(name: impl Into, meta: EntryMeta) -> Self { + Self { + name: name.into(), + kind: EntryKind::Directory, + meta, + } + } + + #[must_use] + pub fn anti(name: impl Into, meta: EntryMeta) -> Self { + Self { + name: name.into(), + kind: EntryKind::Anti, + meta, + } + } +} + +#[derive(Clone)] +pub(crate) struct WriteEntry { + pub name: String, + pub kind: EntryKind, + pub meta: EntryMeta, + pub has_stream: bool, + pub data: Option>, + pub folder_id: usize, +} + +pub(crate) struct CompletedFolder { + pub file_indices: Vec, + pub pack_size: u64, + pub coder_info: Vec, + pub coder_unpack_sizes: Vec, + pub file_sizes: Vec, + pub file_crcs: Vec, +} diff --git a/tests/create_parity_audit_test.rs b/tests/create_parity_audit_test.rs new file mode 100644 index 0000000..ed9ceb5 --- /dev/null +++ b/tests/create_parity_audit_test.rs @@ -0,0 +1,158 @@ +#![allow(clippy::pedantic)] + +mod support; + +use support::{extract_with_p7zip, run_7z_checked}; + +fn write_payload(dir: &std::path::Path) { + let data = (0u8..=255).cycle().take(16 * 1024).collect::>(); + std::fs::write(dir.join("payload.bin"), data).unwrap(); + std::fs::write(dir.join("notes.txt"), b"alpha\nbravo\ncharlie\n".repeat(64)).unwrap(); +} + +#[test] +fn create_parity_audit_p7zip_compression_switches_open_with_r7z() { + let cases: &[(&str, &[&str], &[u8], Option)] = &[ + ("mx0", &["-mx0"], &[0x00], None), + ("mx1", &["-mx1"], &[0x21], Some(1)), + ("mx3", &["-mx3"], &[0x21], Some(1)), + ("mx5", &["-mx5"], &[0x21], Some(1)), + ("mx7", &["-mx7"], &[0x21], Some(1)), + ("mx9", &["-mx9"], &[0x21], Some(1)), + ("lzma", &["-m0=LZMA"], &[0x03, 0x01, 0x01], Some(5)), + ("lzma2", &["-m0=LZMA2"], &[0x21], Some(1)), + ("dict", &["-md=1m"], &[0x21], Some(1)), + ("fast_bytes", &["-mfb=32"], &[0x21], Some(1)), + ("dict_fb", &["-m0=LZMA2:d=1m:fb=32"], &[0x21], Some(1)), + ("solid_off", &["-ms=off"], &[0x21], Some(1)), + ("solid_on", &["-ms=on"], &[0x21], Some(1)), + ("solid_limit", &["-ms=1f"], &[0x21], Some(1)), + ]; + + for (label, args, expected_codec, expected_props_len) in cases { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + write_payload(dir); + let archive_path = dir.join(format!("{label}.7z")); + let mut argv = vec![ + "a", + archive_path.to_str().unwrap(), + "payload.bin", + "notes.txt", + ]; + argv.extend_from_slice(args); + run_7z_checked(&argv, dir); + + let archive = r7z::Archive::open(&archive_path) + .unwrap_or_else(|err| panic!("r7z failed to open {label}: {err}")); + assert_eq!(archive.num_files(), 2, "{label}"); + let unpack_info = archive + .streams_info() + .unwrap() + .unpack_info + .as_ref() + .unwrap_or_else(|| panic!("missing unpack info for {label}")); + assert!(unpack_info.num_folders >= 1, "{label}"); + let folder = unpack_info.parse_folder(0).unwrap(); + let coder = folder + .coders + .first() + .unwrap_or_else(|| panic!("missing coder for {label}")); + assert_eq!(coder.codec_id.as_slice(), *expected_codec, "{label}"); + assert_eq!( + coder.properties.as_deref().map(<[u8]>::len), + *expected_props_len, + "{label}" + ); + + let names = archive.files_info().unwrap().names().collect::>(); + let payload_idx = names + .iter() + .position(|name| name == "payload.bin") + .unwrap_or_else(|| panic!("payload.bin missing from {label}: {names:?}")); + assert_eq!( + archive.extract_to_memory(payload_idx).unwrap(), + std::fs::read(dir.join("payload.bin")).unwrap(), + "{label}" + ); + } +} + +#[test] +fn create_parity_audit_p7zip_volumes_concatenate_to_unsplit_archive() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + write_payload(dir); + + let full = dir.join("full.7z"); + run_7z_checked(&["a", full.to_str().unwrap(), "payload.bin", "-mx0"], dir); + + let split = dir.join("split.7z"); + run_7z_checked( + &["a", split.to_str().unwrap(), "payload.bin", "-mx0", "-v2k"], + dir, + ); + let mut joined = Vec::new(); + for idx in 1.. { + let path = dir.join(format!("split.7z.{idx:03}")); + if !path.exists() { + break; + } + joined.extend_from_slice(&std::fs::read(path).unwrap()); + } + assert_eq!(joined, std::fs::read(&full).unwrap()); + + let list = run_7z_checked(&["l", "split.7z.001"], dir); + let stdout = String::from_utf8_lossy(&list.stdout); + assert!(stdout.contains("Type = Split")); + + let out = dir.join("out"); + extract_with_p7zip(dir, &dir.join("split.7z.001"), &out); + assert_eq!( + std::fs::read(out.join("payload.bin")).unwrap(), + std::fs::read(dir.join("payload.bin")).unwrap() + ); +} + +#[cfg(unix)] +#[test] +fn create_parity_audit_p7zip_link_payloads_and_metadata() { + use std::os::unix::fs::symlink; + + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + std::fs::write(dir.join("target.txt"), b"target").unwrap(); + symlink("target.txt", dir.join("link.txt")).unwrap(); + std::fs::hard_link(dir.join("target.txt"), dir.join("hard.txt")).unwrap(); + + let archive_path = dir.join("links.7z"); + run_7z_checked( + &[ + "a", + archive_path.to_str().unwrap(), + "target.txt", + "link.txt", + "hard.txt", + "-snl", + "-snh", + ], + dir, + ); + + let archive = r7z::Archive::open(&archive_path).unwrap(); + let fi = archive.files_info().unwrap(); + let names = fi.names().collect::>(); + let link_idx = names.iter().position(|name| name == "link.txt").unwrap(); + let hard_idx = names.iter().position(|name| name == "hard.txt").unwrap(); + assert!(matches!( + fi.entry_type(link_idx), + r7z::EntryType::File | r7z::EntryType::Symlink + )); + assert_eq!(fi.entry_type(hard_idx), r7z::EntryType::File); + assert_eq!(archive.extract_to_memory(link_idx).unwrap(), b"target.txt"); + assert_eq!( + archive.symlink_target(link_idx).unwrap().as_deref(), + fi.is_symlink(link_idx).then_some("target.txt") + ); + assert_eq!(archive.extract_to_memory(hard_idx).unwrap(), b"target"); +} diff --git a/tests/file_backed_open_test.rs b/tests/file_backed_open_test.rs new file mode 100644 index 0000000..7cdf00f --- /dev/null +++ b/tests/file_backed_open_test.rs @@ -0,0 +1,231 @@ +use std::alloc::{GlobalAlloc, Layout, System}; +use std::io::{Cursor, Read, Seek, SeekFrom}; +use std::path::Path; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +struct CountingAlloc; + +static ALLOCATED_BYTES: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for CountingAlloc { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc(layout) }; + if !ptr.is_null() { + ALLOCATED_BYTES.fetch_add(layout.size(), Ordering::Relaxed); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new_ptr = unsafe { System.realloc(ptr, layout, new_size) }; + if !new_ptr.is_null() && new_size > layout.size() { + ALLOCATED_BYTES.fetch_add(new_size - layout.size(), Ordering::Relaxed); + } + new_ptr + } +} + +#[global_allocator] +static GLOBAL: CountingAlloc = CountingAlloc; + +fn reset_allocated_bytes() { + ALLOCATED_BYTES.store(0, Ordering::Relaxed); +} + +fn allocated_bytes() -> usize { + ALLOCATED_BYTES.load(Ordering::Relaxed) +} + +#[test] +fn from_reader_accepts_cursor() { + let bytes = r7z::ArchiveBuilder::new() + .add_file("payload.txt", b"cursor-backed") + .build() + .unwrap(); + + let archive = r7z::Archive::from_reader(Cursor::new(bytes)).unwrap(); + + assert_eq!(archive.extract_to_memory(0).unwrap(), b"cursor-backed"); +} + +#[test] +fn open_mmap_parses_fixture() { + let archive = r7z::Archive::open_with_options( + Path::new("tests/fixtures/test_1.7z"), + r7z::ArchiveOpenOptions { + storage_mode: r7z::ArchiveStorageMode::Mmap, + ..Default::default() + }, + ) + .unwrap(); + + assert!(archive.num_files() > 0); +} + +#[test] +fn open_seek_mode_parses_fixture() { + let archive = r7z::Archive::open_with_options( + Path::new("tests/fixtures/test_1.7z"), + r7z::ArchiveOpenOptions { + storage_mode: r7z::ArchiveStorageMode::Seek, + ..Default::default() + }, + ) + .unwrap(); + + assert!(archive.num_files() > 0); +} + +#[test] +fn sparse_large_seek_open_does_not_read_whole_file() { + let tmp = tempfile::tempdir().unwrap(); + let archive_path = tmp.path().join("sparse.7z"); + let bytes = r7z::ArchiveBuilder::new() + .add_file("payload.txt", b"sparse") + .build() + .unwrap(); + std::fs::write(&archive_path, bytes).unwrap(); + let file = std::fs::OpenOptions::new() + .write(true) + .open(&archive_path) + .unwrap(); + file.set_len(256 * 1024 * 1024).unwrap(); + drop(file); + + reset_allocated_bytes(); + let archive = r7z::Archive::open_with_options( + &archive_path, + r7z::ArchiveOpenOptions { + storage_mode: r7z::ArchiveStorageMode::Seek, + ..Default::default() + }, + ) + .unwrap(); + let allocated = allocated_bytes(); + + assert_eq!(archive.num_files(), 1); + assert!( + allocated < 8 * 1024 * 1024, + "seek-backed open allocated {allocated} bytes" + ); +} + +#[test] +fn metadata_limit_rejects_large_next_header() { + let tmp = tempfile::tempdir().unwrap(); + let archive_path = tmp.path().join("limited.7z"); + let bytes = r7z::ArchiveBuilder::new() + .add_file("payload.txt", b"metadata") + .build() + .unwrap(); + std::fs::write(&archive_path, bytes).unwrap(); + + let err = match r7z::Archive::open_with_options( + &archive_path, + r7z::ArchiveOpenOptions { + max_metadata_bytes: 1, + storage_mode: r7z::ArchiveStorageMode::Seek, + }, + ) { + Ok(_) => panic!("archive opened despite metadata limit"), + Err(err) => err, + }; + + assert!(matches!(err, r7z::R7zError::LimitExceeded("metadata"))); +} + +#[test] +fn metadata_limit_rejects_large_decoded_header() { + let tmp = tempfile::tempdir().unwrap(); + let archive_path = tmp.path().join("decoded-header-limited.7z"); + let mut builder = r7z::ArchiveBuilder::new(); + for i in 0..128 { + builder = builder.add_file(&format!("entry-{i:03}.txt"), b"x"); + } + let bytes = builder.build().unwrap(); + let next_header_size = u64::from_le_bytes(bytes[20..28].try_into().unwrap()); + std::fs::write(&archive_path, bytes).unwrap(); + + let err = match r7z::Archive::open_with_options( + &archive_path, + r7z::ArchiveOpenOptions { + max_metadata_bytes: next_header_size + 16, + storage_mode: r7z::ArchiveStorageMode::Seek, + }, + ) { + Ok(_) => panic!("archive opened despite decoded metadata limit"), + Err(err) => err, + }; + + assert!(matches!(err, r7z::R7zError::LimitExceeded("metadata"))); +} + +#[test] +fn extract_to_writer_from_seek_source_matches_from_bytes() { + let bytes = r7z::ArchiveBuilder::new() + .compression(r7z::Codec::Lzma2) + .add_file("payload.txt", b"seek-backed extract") + .build() + .unwrap(); + let from_bytes = r7z::Archive::from_bytes(bytes.clone().into()).unwrap(); + let from_reader = r7z::Archive::from_reader(Cursor::new(bytes)).unwrap(); + + let mut expected = Vec::new(); + let mut actual = Vec::new(); + from_bytes.extract_to_writer(0, &mut expected).unwrap(); + from_reader.extract_to_writer(0, &mut actual).unwrap(); + + assert_eq!(actual, expected); +} + +#[test] +fn extract_to_writer_non_aes_does_not_read_packed_stream_in_one_request() { + let payload = vec![0xA5; 2 * 1024 * 1024]; + let bytes = r7z::ArchiveBuilder::new() + .compression(r7z::Codec::Copy) + .add_file("payload.bin", &payload) + .build() + .unwrap(); + let max_read_request = Arc::new(AtomicUsize::new(0)); + let reader = TrackingReader { + inner: Cursor::new(bytes), + max_read_request: Arc::clone(&max_read_request), + }; + let archive = r7z::Archive::from_reader(reader).unwrap(); + + max_read_request.store(0, Ordering::Relaxed); + let mut out = Vec::new(); + archive.extract_to_writer(0, &mut out).unwrap(); + + assert_eq!(out, payload); + assert!( + max_read_request.load(Ordering::Relaxed) <= 64 * 1024, + "extract read the packed stream in a large request" + ); +} + +struct TrackingReader { + inner: Cursor>, + max_read_request: Arc, +} + +impl Read for TrackingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.max_read_request + .fetch_max(buf.len(), Ordering::Relaxed); + self.inner.read(buf) + } +} + +impl Seek for TrackingReader { + fn seek(&mut self, pos: SeekFrom) -> std::io::Result { + self.inner.seek(pos) + } +} diff --git a/tests/interop_test.rs b/tests/interop_test.rs index 88f1a00..1bdfd59 100644 --- a/tests/interop_test.rs +++ b/tests/interop_test.rs @@ -1,3 +1,5 @@ +#![allow(clippy::pedantic)] + //! Interop tests: create archives with p7zip, extract with r7z, byte-compare. //! //! These tests require `7z` (p7zip) to be available in PATH or via nix-shell. diff --git a/tests/interop_write_test.rs b/tests/interop_write_test.rs index 9f74836..61f5642 100644 --- a/tests/interop_write_test.rs +++ b/tests/interop_write_test.rs @@ -1,6 +1,9 @@ +#![allow(clippy::pedantic)] + //! Write-interop tests: create archives with r7z, extract with p7zip, byte-compare. mod support; +use std::num::NonZeroU64; use std::path::{Path, PathBuf}; use support::{assert_extracted_files, extract_with_p7zip, list_with_p7zip, run_7z}; @@ -39,6 +42,40 @@ fn executable_payload(size: usize) -> Vec { data } +fn filetime_from_unix_secs(secs: u64) -> u64 { + (secs + 11_644_473_600) * 10_000_000 +} + +fn assert_default_aes_properties(props: &[u8]) { + assert_eq!(props.len(), 18); + assert_eq!(props[0] & 0x3F, 19); + assert_eq!(props[0] & 0x80, 0, "default AES salt should be absent"); + assert_eq!(props[0] & 0x40, 0x40, "default AES IV should be present"); + assert_eq!(props[1] >> 4, 0, "default AES salt length should be zero"); + assert_eq!(props[1] & 0x0F, 15, "default AES IV length should be 16"); + assert!( + props[2..].iter().any(|&b| b != 0), + "generated AES IV should not be all zero" + ); +} + +fn assert_salted_aes_properties(props: &[u8]) { + assert_eq!(props.len(), 18); + assert_eq!(props[0] & 0x3F, 19); + assert_eq!(props[0] & 0x80, 0x80, "AES salt should be present"); + assert_eq!(props[0] & 0x40, 0x40, "AES IV should be present"); + assert_eq!(props[1] >> 4, 7, "AES salt length should be 8"); + assert_eq!(props[1] & 0x0F, 7, "AES IV length should be 8"); + assert!( + props[2..10].iter().any(|&b| b != 0), + "generated AES salt should not be all zero" + ); + assert!( + props[10..].iter().any(|&b| b != 0), + "generated AES IV should not be all zero" + ); +} + fn write_builder_archive(archive_path: &Path, codec: r7z::Codec, files: &[(PathBuf, Vec)]) { let mut builder = r7z::ArchiveBuilder::new().compression(codec); for (name, data) in files { @@ -51,7 +88,7 @@ fn write_builder_archive(archive_path: &Path, codec: r7z::Codec, files: &[(PathB fn write_writer_archive(archive_path: &Path, codec: r7z::Codec, files: &[(PathBuf, Vec)]) { let file = std::fs::File::create(archive_path).unwrap(); - let mut writer = r7z::ArchiveWriter::new(file) + let mut writer = r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()) .expect("ArchiveWriter::new failed") .compression(codec); for (idx, (name, data)) in files.iter().enumerate() { @@ -270,7 +307,8 @@ fn archive_writer_single_folder_r7z_reads() { ]; let mut buf = std::io::Cursor::new(Vec::new()); - let mut w = r7z::ArchiveWriter::new(&mut buf).expect("new failed"); + let mut w = + r7z::ArchiveWriter::new(&mut buf, r7z::ArchiveOptions::default()).expect("new failed"); for (name, data) in &files { w.append(name, *data).expect("append failed"); } @@ -299,7 +337,8 @@ fn archive_writer_multi_folder_r7z_reads() { ]; let mut buf = std::io::Cursor::new(Vec::new()); - let mut w = r7z::ArchiveWriter::new(&mut buf).expect("new failed"); + let mut w = + r7z::ArchiveWriter::new(&mut buf, r7z::ArchiveOptions::default()).expect("new failed"); for (name, data) in &folder0 { w.append(name, *data).expect("append failed"); } @@ -335,7 +374,7 @@ fn archive_writer_multi_folder_p7zip_reads() { let archive_path = dir.join("writer_multi.7z"); let file = std::fs::File::create(&archive_path).unwrap(); - let mut w = r7z::ArchiveWriter::new(file).expect("new failed"); + let mut w = r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()).expect("new failed"); for (name, data) in &folder0 { w.append(name, *data).expect("append failed"); } @@ -393,11 +432,11 @@ fn archive_writer_mtime_r7z_reads() { let ts = UNIX_EPOCH + Duration::from_secs(1_710_504_000); let meta = r7z::EntryMeta { mtime: Some(ts), - unix_mode: None, + ..Default::default() }; let mut buf = std::io::Cursor::new(Vec::new()); - let mut w = r7z::ArchiveWriter::new(&mut buf).unwrap(); + let mut w = r7z::ArchiveWriter::new(&mut buf, r7z::ArchiveOptions::default()).unwrap(); w.append_entry("ts.txt", b"timestamp test".as_ref(), meta) .unwrap(); w.finish().unwrap(); @@ -411,18 +450,15 @@ fn archive_writer_mtime_r7z_reads() { assert_eq!(fi.mtimes.first().copied().flatten(), Some(expected_ft)); } -/// [`ArchiveWriter::append_entry`] with unix_mode: attributes survive write → read. +/// [`ArchiveWriter::append_entry`] with Unix-mode attributes survives write -> read. #[test] fn archive_writer_unix_mode_r7z_reads() { // Regular file, rw-r--r-- = 0o100644 let mode: u32 = 0o100_644; - let meta = r7z::EntryMeta { - mtime: None, - unix_mode: Some(mode), - }; + let meta = r7z::EntryMeta::from_unix_mode(mode); let mut buf = std::io::Cursor::new(Vec::new()); - let mut w = r7z::ArchiveWriter::new(&mut buf).unwrap(); + let mut w = r7z::ArchiveWriter::new(&mut buf, r7z::ArchiveOptions::default()).unwrap(); w.append_entry("perms.txt", b"permissions test".as_ref(), meta) .unwrap(); w.finish().unwrap(); @@ -436,6 +472,43 @@ fn archive_writer_unix_mode_r7z_reads() { assert_eq!(attrs & 0xFFFF, 0x20); } +#[test] +fn archive_builder_full_metadata_r7z_reads() { + use std::time::{Duration, UNIX_EPOCH}; + + let ctime_secs = 1_577_836_800; // 2020-01-01T00:00:00Z + let atime_secs = 1_609_459_200; // 2021-01-01T00:00:00Z + let mtime_secs = 1_640_995_200; // 2022-01-01T00:00:00Z + let data_meta = r7z::EntryMeta { + ctime: Some(UNIX_EPOCH + Duration::from_secs(ctime_secs)), + atime: Some(UNIX_EPOCH + Duration::from_secs(atime_secs)), + mtime: Some(UNIX_EPOCH + Duration::from_secs(mtime_secs)), + start_pos: Some(123), + ..r7z::EntryMeta::from_unix_mode(0o100_640) + }; + let plain_meta = r7z::EntryMeta::archive_file(); + + let bytes = r7z::ArchiveBuilder::new() + .add_file_entry("meta.txt", b"metadata", data_meta) + .add_file_entry("plain.txt", b"plain", plain_meta) + .build() + .expect("build failed"); + + let archive = r7z::Archive::from_bytes(bytes.into()).expect("from_bytes failed"); + let fi = archive.files_info().unwrap(); + + assert_eq!(fi.ctimes[0], Some(filetime_from_unix_secs(ctime_secs))); + assert_eq!(fi.ctimes[1], None); + assert_eq!(fi.atimes[0], Some(filetime_from_unix_secs(atime_secs))); + assert_eq!(fi.atimes[1], None); + assert_eq!(fi.mtimes[0], Some(filetime_from_unix_secs(mtime_secs))); + assert_eq!(fi.mtimes[1], None); + assert_eq!(fi.start_positions[0], Some(123)); + assert_eq!(fi.start_positions[1], None); + assert_eq!(fi.attributes[0], Some((0o100_640 << 16) | 0x20)); + assert_eq!(fi.attributes[1], Some(0x20)); +} + /// p7zip lists a non-epoch timestamp for an archive written with mtime via [`ArchiveWriter`]. #[test] fn archive_writer_mtime_p7zip_reads() { @@ -447,12 +520,12 @@ fn archive_writer_mtime_p7zip_reads() { let ts = UNIX_EPOCH + Duration::from_secs(1_710_504_000); // 2024-03-15T12:00:00Z let meta = r7z::EntryMeta { mtime: Some(ts), - unix_mode: None, + ..Default::default() }; let archive_path = dir.join("ts.7z"); let file = std::fs::File::create(&archive_path).unwrap(); - let mut w = r7z::ArchiveWriter::new(file).unwrap(); + let mut w = r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()).unwrap(); w.append_entry("ts.txt", b"timestamp data".as_ref(), meta) .unwrap(); w.finish().unwrap(); @@ -557,7 +630,7 @@ fn archive_writer_bcj_lzma2_p7zip_reads() { let archive_path = dir.join("bcj_writer.7z"); let file = std::fs::File::create(&archive_path).unwrap(); - let mut w = r7z::ArchiveWriter::new(file) + let mut w = r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()) .unwrap() .compression(r7z::Codec::Lzma2Bcj); w.append("code.bin", &mut data.as_slice()).unwrap(); @@ -654,3 +727,1127 @@ fn build_streaming_lzma2_p7zip_extracts_and_lists_method() { r7z::build_streaming(entries, output).expect("build_streaming failed"); assert_p7zip_extracts_archive(dir, &archive_path, &files, &["LZMA2"]); } + +#[test] +fn build_streaming_with_options_copy_p7zip_extracts_and_lists_method() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let files = parity_files(); + let archive_path = dir.join("streaming_copy.7z"); + let output = std::fs::File::create(&archive_path).unwrap(); + let entries = files + .iter() + .map(|(name, data)| (name.to_string_lossy().into_owned(), data.as_slice())); + let options = r7z::ArchiveOptions { + codec: r7z::Codec::Copy, + ..Default::default() + }; + + r7z::build_streaming_with_options(entries, output, options) + .expect("build_streaming_with_options failed"); + assert_p7zip_extracts_archive(dir, &archive_path, &files, &["Copy"]); +} + +#[test] +fn build_streaming_with_options_lzma_p7zip_extracts_and_lists_method() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let files = parity_files(); + let archive_path = dir.join("streaming_lzma.7z"); + let output = std::fs::File::create(&archive_path).unwrap(); + let entries = files + .iter() + .map(|(name, data)| (name.to_string_lossy().into_owned(), data.as_slice())); + let options = r7z::ArchiveOptions { + codec: r7z::Codec::Lzma, + ..Default::default() + }; + + r7z::build_streaming_with_options(entries, output, options) + .expect("build_streaming_with_options failed"); + assert_p7zip_extracts_archive(dir, &archive_path, &files, &["LZMA"]); +} + +#[test] +fn build_streaming_with_options_bcj_lzma2_p7zip_extracts_and_lists_method() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let files = executable_files(); + let archive_path = dir.join("streaming_bcj_lzma2.7z"); + let output = std::fs::File::create(&archive_path).unwrap(); + let entries = files + .iter() + .map(|(name, data)| (name.to_string_lossy().into_owned(), data.as_slice())); + let options = r7z::ArchiveOptions { + codec: r7z::Codec::Lzma2Bcj, + ..Default::default() + }; + + r7z::build_streaming_with_options(entries, output, options) + .expect("build_streaming_with_options failed"); + assert_p7zip_extracts_archive(dir, &archive_path, &files, &["BCJ", "LZMA2"]); +} + +#[test] +fn archive_builder_default_is_lzma2_and_uses_encoded_header_for_multi_entry() { + let bytes = r7z::ArchiveBuilder::new() + .add_file("a.txt", b"alpha") + .add_file("b.txt", b"bravo") + .build() + .expect("build failed"); + + let archive = r7z::Archive::from_bytes(bytes.into()).expect("from_bytes failed"); + assert!(archive.encoded_header.is_some()); + let ui = archive + .streams_info() + .unwrap() + .unpack_info + .as_ref() + .unwrap(); + let folder = ui.parse_folder(0).unwrap(); + assert_eq!(folder.coders[0].codec_id.as_slice(), r7z::CODEC_LZMA2); +} + +#[test] +fn archive_builder_header_modes_are_honored() { + let single_default = r7z::ArchiveBuilder::new() + .add_file("single.txt", b"one") + .build() + .expect("build failed"); + let archive = r7z::Archive::from_bytes(single_default.into()).expect("from_bytes failed"); + assert!(archive.encoded_header.is_none()); + + let encoded = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + header_mode: r7z::HeaderMode::Encoded, + ..Default::default() + }) + .add_file("single.txt", b"one") + .build() + .expect("build failed"); + let archive = r7z::Archive::from_bytes(encoded.into()).expect("from_bytes failed"); + assert!(archive.encoded_header.is_some()); + + let plain = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + header_mode: r7z::HeaderMode::Plain, + ..Default::default() + }) + .add_file("a.txt", b"alpha") + .add_file("b.txt", b"bravo") + .build() + .expect("build failed"); + let archive = r7z::Archive::from_bytes(plain.into()).expect("from_bytes failed"); + assert!(archive.encoded_header.is_none()); +} + +#[test] +fn archive_builder_copy_p7zip_extracts_and_lists_method() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let files = parity_files(); + let archive_path = dir.join("copy.7z"); + + write_builder_archive(&archive_path, r7z::Codec::Copy, &files); + assert_p7zip_extracts_archive(dir, &archive_path, &files, &["Copy"]); +} + +#[test] +fn archive_writer_copy_streams_payload_before_finish() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("writer_copy_streamed.7z"); + let payload = vec![0xA7; 128 * 1024]; + let file = std::fs::File::create(&archive_path).unwrap(); + let mut writer = r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()) + .expect("new failed") + .compression(r7z::Codec::Copy); + + writer + .append_file( + "streamed.bin", + payload.as_slice(), + r7z::EntryMeta::archive_file(), + ) + .expect("append failed"); + assert!( + std::fs::metadata(&archive_path).unwrap().len() > payload.len() as u64, + "Copy writer should write payload bytes before finish" + ); + + writer.finish().expect("finish failed"); + assert_p7zip_extracts_archive( + dir, + &archive_path, + &[(PathBuf::from("streamed.bin"), payload)], + &["Copy"], + ); +} + +#[test] +fn archive_writer_lzma2_streams_payload_before_finish() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("writer_lzma2_streamed.7z"); + let payload = (0u8..=255).cycle().take(1024 * 1024).collect::>(); + let file = std::fs::File::create(&archive_path).unwrap(); + let mut writer = + r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()).expect("new failed"); + + writer + .append_file( + "streamed.bin", + payload.as_slice(), + r7z::EntryMeta::archive_file(), + ) + .expect("append failed"); + writer.new_folder().expect("new_folder failed"); + assert!( + std::fs::metadata(&archive_path).unwrap().len() > 32, + "LZMA2 writer should emit compressed payload bytes after sealing a folder" + ); + + writer.finish().expect("finish failed"); + assert_p7zip_extracts_archive( + dir, + &archive_path, + &[(PathBuf::from("streamed.bin"), payload)], + &["LZMA2"], + ); +} + +#[test] +fn archive_writer_lzma_streams_payload_before_finish() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("writer_lzma_streamed.7z"); + let payload = (0u8..=255) + .rev() + .cycle() + .take(1024 * 1024) + .collect::>(); + let file = std::fs::File::create(&archive_path).unwrap(); + let mut writer = r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()) + .expect("new failed") + .compression(r7z::Codec::Lzma); + + writer + .append_file( + "streamed.bin", + payload.as_slice(), + r7z::EntryMeta::archive_file(), + ) + .expect("append failed"); + writer.new_folder().expect("new_folder failed"); + assert!( + std::fs::metadata(&archive_path).unwrap().len() > 32, + "LZMA writer should emit compressed payload bytes after sealing a folder" + ); + + writer.finish().expect("finish failed"); + assert_p7zip_extracts_archive( + dir, + &archive_path, + &[(PathBuf::from("streamed.bin"), payload)], + &["LZMA"], + ); +} + +#[test] +fn archive_writer_bcj_lzma2_streams_payload_before_finish() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("writer_bcj_lzma2_streamed.7z"); + let payload = executable_payload(1024 * 1024); + let file = std::fs::File::create(&archive_path).unwrap(); + let mut writer = r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()) + .expect("new failed") + .compression(r7z::Codec::Lzma2Bcj); + + writer + .append_file( + "streamed.exe", + payload.as_slice(), + r7z::EntryMeta::archive_file(), + ) + .expect("append failed"); + writer.new_folder().expect("new_folder failed"); + assert!( + std::fs::metadata(&archive_path).unwrap().len() > 32, + "BCJ+LZMA2 writer should emit compressed payload bytes after sealing a folder" + ); + + writer.finish().expect("finish failed"); + assert_p7zip_extracts_archive( + dir, + &archive_path, + &[(PathBuf::from("streamed.exe"), payload)], + &["BCJ", "LZMA2"], + ); +} + +#[test] +fn archive_writer_mixed_empty_entries_preserve_order_and_folder_boundaries() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("writer_mixed.7z"); + let file = std::fs::File::create(&archive_path).unwrap(); + let mut writer = + r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()).expect("new failed"); + writer + .append_file("a.txt", b"alpha".as_slice(), r7z::EntryMeta::archive_file()) + .expect("append file failed"); + writer.new_folder().expect("new_folder failed"); + writer + .append_directory("nested", r7z::EntryMeta::directory_unix_mode(0o040_755)) + .expect("append directory failed"); + writer + .append_empty_file("nested/empty.txt", r7z::EntryMeta::archive_file()) + .expect("append empty file failed"); + writer + .append_anti_item("removed.txt", r7z::EntryMeta::default()) + .expect("append anti failed"); + writer.new_folder().expect("new_folder failed"); + writer + .append_file( + "nested/b.txt", + b"bravo".as_slice(), + r7z::EntryMeta::archive_file(), + ) + .expect("append file failed"); + writer.finish().expect("finish failed"); + + let archive = r7z::Archive::open(&archive_path).unwrap(); + let fi = archive.files_info().unwrap(); + assert_eq!(archive.num_files(), 5); + assert_eq!(fi.name(0).unwrap(), "a.txt"); + assert_eq!(fi.name(1).unwrap(), "nested"); + assert_eq!(fi.name(2).unwrap(), "nested/empty.txt"); + assert_eq!(fi.name(3).unwrap(), "removed.txt"); + assert_eq!(fi.name(4).unwrap(), "nested/b.txt"); + assert!(fi.is_directory(1)); + assert!(fi.is_empty_file(2)); + assert!(fi.is_anti(3)); + let unpack_info = archive + .streams_info() + .unwrap() + .unpack_info + .as_ref() + .unwrap(); + assert_eq!(unpack_info.num_folders, 2); + assert_eq!(archive.extract_to_memory(0).unwrap(), b"alpha"); + assert_eq!(archive.extract_to_memory(2).unwrap(), b""); + assert_eq!(archive.extract_to_memory(4).unwrap(), b"bravo"); + + let out_dir = dir.join("out"); + extract_with_p7zip(dir, &archive_path, &out_dir); + assert_eq!(std::fs::read(out_dir.join("a.txt")).unwrap(), b"alpha"); + assert!(out_dir.join("nested").is_dir()); + assert_eq!( + std::fs::read(out_dir.join("nested/empty.txt")).unwrap(), + b"" + ); + assert_eq!( + std::fs::read(out_dir.join("nested/b.txt")).unwrap(), + b"bravo" + ); +} + +#[test] +fn archive_entry_helpers_round_trip_and_validate_stream_kind() { + let builder = r7z::ArchiveBuilder::new() + .add_entry( + r7z::ArchiveEntry::directory("dir", r7z::EntryMeta::default()), + None, + ) + .unwrap() + .add_entry( + r7z::ArchiveEntry::file("dir/data.txt", r7z::EntryMeta::archive_file()), + Some(b"hello"), + ) + .unwrap() + .add_entry( + r7z::ArchiveEntry::file("dir/empty.txt", r7z::EntryMeta::default()), + None, + ) + .unwrap() + .add_entry( + r7z::ArchiveEntry::anti("removed.txt", r7z::EntryMeta::default()), + None, + ) + .unwrap(); + let archive = r7z::Archive::from_bytes(builder.build().unwrap().into()).unwrap(); + let fi = archive.files_info().unwrap(); + assert!(fi.is_directory(0)); + assert_eq!(archive.extract_to_memory(1).unwrap(), b"hello"); + assert!(fi.is_empty_file(2)); + assert!(fi.is_anti(3)); + + let invalid = r7z::ArchiveBuilder::new().add_entry( + r7z::ArchiveEntry::directory("bad", r7z::EntryMeta::default()), + Some(b"not allowed"), + ); + assert!(matches!(invalid, Err(r7z::R7zError::InvalidOptions(_)))); + + let mut buf = std::io::Cursor::new(Vec::new()); + let mut writer = + r7z::ArchiveWriter::new(&mut buf, r7z::ArchiveOptions::default()).expect("new failed"); + writer + .append_empty_entry(r7z::ArchiveEntry::directory( + "dir", + r7z::EntryMeta::default(), + )) + .unwrap(); + writer + .append_archive_entry( + r7z::ArchiveEntry::file("dir/data.txt", r7z::EntryMeta::archive_file()), + b"hello".as_slice(), + ) + .unwrap(); + writer + .append_empty_entry(r7z::ArchiveEntry::file( + "dir/empty.txt", + r7z::EntryMeta::default(), + )) + .unwrap(); + writer + .append_empty_entry(r7z::ArchiveEntry::anti( + "removed.txt", + r7z::EntryMeta::default(), + )) + .unwrap(); + writer.finish().unwrap(); + + let archive = r7z::Archive::from_bytes(buf.into_inner().into()).unwrap(); + let fi = archive.files_info().unwrap(); + assert!(fi.is_directory(0)); + assert_eq!(archive.extract_to_memory(1).unwrap(), b"hello"); + assert!(fi.is_empty_file(2)); + assert!(fi.is_anti(3)); +} + +#[test] +fn archive_builder_empty_directory_and_anti_items_round_trip_and_p7zip_lists() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("specials.7z"); + let bytes = r7z::ArchiveBuilder::new() + .add_directory("nested", r7z::EntryMeta::default()) + .add_empty_file("nested/empty.txt", r7z::EntryMeta::default()) + .add_file("nested/data.txt", b"payload") + .add_anti_item("deleted.txt", r7z::EntryMeta::default()) + .build() + .expect("build failed"); + std::fs::write(&archive_path, bytes).unwrap(); + + let archive = r7z::Archive::open(&archive_path).unwrap(); + let fi = archive.files_info().unwrap(); + assert!(fi.is_directory(0)); + assert!(fi.is_empty_file(1)); + assert!(fi.is_anti(3)); + assert!(!fi.is_directory(3)); + assert_eq!(archive.extract_to_memory(2).unwrap(), b"payload"); + + let listing = list_with_p7zip(dir, &archive_path); + assert!(listing.contains("nested/empty.txt")); + assert!(listing.contains("deleted.txt")); + + let out_dir = dir.join("out"); + extract_with_p7zip(dir, &archive_path, &out_dir); + assert!(out_dir.join("nested").is_dir()); + assert_eq!( + std::fs::read(out_dir.join("nested/empty.txt")).unwrap(), + b"" + ); + assert_eq!( + std::fs::read(out_dir.join("nested/data.txt")).unwrap(), + b"payload" + ); +} + +#[test] +fn archive_builder_empty_only_p7zip_extracts_and_r7z_reads() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("empty_only.7z"); + let bytes = r7z::ArchiveBuilder::new() + .add_directory("emptydir", r7z::EntryMeta::directory_unix_mode(0o040_755)) + .add_empty_file("emptydir/empty.txt", r7z::EntryMeta::archive_file()) + .add_anti_item("removed.txt", r7z::EntryMeta::default()) + .build() + .expect("build failed"); + std::fs::write(&archive_path, bytes).unwrap(); + + let archive = r7z::Archive::open(&archive_path).unwrap(); + assert!(archive.streams_info().is_none()); + let fi = archive.files_info().unwrap(); + assert_eq!(archive.num_files(), 3); + assert_eq!(fi.name(0).unwrap(), "emptydir"); + assert_eq!(fi.name(1).unwrap(), "emptydir/empty.txt"); + assert_eq!(fi.name(2).unwrap(), "removed.txt"); + assert!(fi.is_directory(0)); + assert!(fi.is_empty_file(1)); + assert!(fi.is_anti(2)); + assert!(!fi.is_directory(2)); + assert_eq!(archive.extract_to_memory(1).unwrap(), b""); + + let listing = list_with_p7zip(dir, &archive_path); + assert!(listing.contains("emptydir")); + assert!(listing.contains("emptydir/empty.txt")); + assert!(listing.contains("removed.txt")); + + let out_dir = dir.join("out"); + extract_with_p7zip(dir, &archive_path, &out_dir); + assert!(out_dir.join("emptydir").is_dir()); + assert_eq!( + std::fs::read(out_dir.join("emptydir/empty.txt")).unwrap(), + b"" + ); +} + +#[test] +fn archive_builder_aes_content_p7zip_and_r7z_extract_with_password() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("aes_content.7z"); + let options = r7z::ArchiveOptions { + encryption: Some(r7z::EncryptionOptions::default_for_password("Secret123")), + ..Default::default() + }; + let bytes = r7z::ArchiveBuilder::new() + .options(options) + .add_file("secret.txt", b"classified") + .build() + .expect("build failed"); + std::fs::write(&archive_path, bytes).unwrap(); + + let archive = r7z::Archive::open(&archive_path).unwrap(); + assert!(matches!( + archive.extract_to_memory(0).unwrap_err(), + r7z::R7zError::PasswordRequired + )); + assert_eq!( + archive + .extract_to_memory_with_password(0, Some("Secret123")) + .unwrap(), + b"classified" + ); + + let out_dir = dir.join("out"); + let out_arg = format!("-o{}", out_dir.to_str().unwrap()); + let out = run_7z( + &[ + "x", + "-y", + "-pSecret123", + archive_path.to_str().unwrap(), + &out_arg, + ], + dir, + ); + assert!( + out.status.success(), + "7z x failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + std::fs::read(out_dir.join("secret.txt")).unwrap(), + b"classified" + ); +} + +#[test] +fn archive_builder_default_aes_properties_match_p7zip_settings() { + let options = r7z::ArchiveOptions { + encryption: Some(r7z::EncryptionOptions::default_for_password("Secret123")), + ..Default::default() + }; + let bytes = r7z::ArchiveBuilder::new() + .options(options) + .add_file("secret.txt", b"classified") + .build() + .expect("build failed"); + + let archive = r7z::Archive::from_bytes(bytes.into()).unwrap(); + let streams = archive.streams_info().unwrap(); + let unpack_info = streams.unpack_info.as_ref().unwrap(); + let folder = unpack_info.parse_folder(0).unwrap(); + let aes_coder = &folder.coders[0]; + assert_eq!(aes_coder.codec_id.as_slice(), r7z::CODEC_AES_256_SHA_256); + assert_default_aes_properties(aes_coder.properties.as_deref().unwrap()); + + let mut enc = r7z::EncryptionOptions::default_for_password("HeaderSecret"); + enc.encrypt_header = true; + let bytes = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }) + .add_file("hidden.txt", b"hidden payload") + .build() + .expect("build failed"); + let archive = r7z::Archive::from_bytes_with_password(bytes.into(), Some("HeaderSecret")) + .expect("from_bytes_with_password failed"); + let encoded_header = archive.encoded_header.as_ref().unwrap(); + let folder = encoded_header.unpack_info.parse_folder(0).unwrap(); + let aes_coder = &folder.coders[0]; + assert_eq!(aes_coder.codec_id.as_slice(), r7z::CODEC_AES_256_SHA_256); + assert_default_aes_properties(aes_coder.properties.as_deref().unwrap()); +} + +#[test] +fn archive_builder_salted_aes_content_p7zip_and_r7z_extract() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("aes_salted.7z"); + let mut enc = r7z::EncryptionOptions::default_for_password("Secret123"); + enc.salt_len = 8; + enc.iv_len = 8; + let bytes = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }) + .add_file("secret.txt", b"salted secret") + .build() + .expect("build failed"); + std::fs::write(&archive_path, bytes).unwrap(); + + let archive = r7z::Archive::open(&archive_path).unwrap(); + let streams = archive.streams_info().unwrap(); + let unpack_info = streams.unpack_info.as_ref().unwrap(); + let folder = unpack_info.parse_folder(0).unwrap(); + let aes_coder = &folder.coders[0]; + assert_eq!(aes_coder.codec_id.as_slice(), r7z::CODEC_AES_256_SHA_256); + assert_salted_aes_properties(aes_coder.properties.as_deref().unwrap()); + assert_eq!( + archive + .extract_to_memory_with_password(0, Some("Secret123")) + .unwrap(), + b"salted secret" + ); + + let out_dir = dir.join("out"); + let out_arg = format!("-o{}", out_dir.to_str().unwrap()); + let out = run_7z( + &[ + "x", + "-y", + "-pSecret123", + archive_path.to_str().unwrap(), + &out_arg, + ], + dir, + ); + assert!( + out.status.success(), + "7z x failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + std::fs::read(out_dir.join("secret.txt")).unwrap(), + b"salted secret" + ); +} + +#[test] +fn archive_builder_salted_aes_encrypted_header_p7zip_and_r7z_extract() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("aes_salted_header.7z"); + let mut enc = r7z::EncryptionOptions::default_for_password("HeaderSecret"); + enc.encrypt_header = true; + enc.salt_len = 8; + enc.iv_len = 8; + let bytes = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }) + .add_file("hidden.txt", b"salted header") + .build() + .expect("build failed"); + std::fs::write(&archive_path, bytes).unwrap(); + + let err = match r7z::Archive::open(&archive_path) { + Ok(_) => panic!("encrypted salted header opened without password"), + Err(err) => err, + }; + assert!(matches!(err, r7z::R7zError::PasswordRequired)); + let archive = r7z::Archive::open_with_password(&archive_path, Some("HeaderSecret")).unwrap(); + let encoded_header = archive.encoded_header.as_ref().unwrap(); + let folder = encoded_header.unpack_info.parse_folder(0).unwrap(); + let aes_coder = &folder.coders[0]; + assert_eq!(aes_coder.codec_id.as_slice(), r7z::CODEC_AES_256_SHA_256); + assert_salted_aes_properties(aes_coder.properties.as_deref().unwrap()); + assert_eq!( + archive + .extract_to_memory_with_password(0, Some("HeaderSecret")) + .unwrap(), + b"salted header" + ); + + let out_dir = dir.join("out"); + let out_arg = format!("-o{}", out_dir.to_str().unwrap()); + let out = run_7z( + &[ + "x", + "-y", + "-pHeaderSecret", + archive_path.to_str().unwrap(), + &out_arg, + ], + dir, + ); + assert!( + out.status.success(), + "7z x failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + std::fs::read(out_dir.join("hidden.txt")).unwrap(), + b"salted header" + ); +} + +#[test] +fn archive_builder_aes_content_copy_and_bcj_p7zip_and_r7z_extract() { + let cases = [ + ( + r7z::Codec::Copy, + "aes_copy.7z", + "raw.bin", + (0u8..=255).cycle().take(4097).collect::>(), + ), + ( + r7z::Codec::Lzma2Bcj, + "aes_bcj.7z", + "bin/app.exe", + executable_payload(4096), + ), + ]; + + for (codec, archive_name, entry_name, payload) in cases { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join(archive_name); + let options = r7z::ArchiveOptions { + codec, + encryption: Some(r7z::EncryptionOptions::default_for_password("Secret123")), + ..Default::default() + }; + let bytes = r7z::ArchiveBuilder::new() + .options(options) + .add_file(entry_name, &payload) + .build() + .expect("build failed"); + std::fs::write(&archive_path, bytes).unwrap(); + + let archive = r7z::Archive::open(&archive_path).unwrap(); + assert_eq!( + archive + .extract_to_memory_with_password(0, Some("Secret123")) + .unwrap(), + payload + ); + + let out_dir = dir.join("out"); + std::fs::create_dir_all(&out_dir).unwrap(); + let out_arg = format!("-o{}", out_dir.to_str().unwrap()); + let out = run_7z( + &[ + "x", + "-y", + "-pSecret123", + archive_path.to_str().unwrap(), + &out_arg, + ], + dir, + ); + assert!( + out.status.success(), + "7z x failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!(std::fs::read(out_dir.join(entry_name)).unwrap(), payload); + } +} + +#[test] +fn archive_builder_rejects_invalid_aes_options() { + let mut enc = r7z::EncryptionOptions::default_for_password("Secret123"); + enc.salt_len = 17; + let err = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }) + .add_file("secret.txt", b"classified") + .build() + .unwrap_err(); + assert!(matches!(err, r7z::R7zError::InvalidOptions(_))); + + let mut enc = r7z::EncryptionOptions::default_for_password("Secret123"); + enc.iv_len = 17; + let err = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }) + .add_file("secret.txt", b"classified") + .build() + .unwrap_err(); + assert!(matches!(err, r7z::R7zError::InvalidOptions(_))); + + let mut enc = r7z::EncryptionOptions::default_for_password("Secret123"); + enc.num_cycles_power = 25; + enc.encrypt_header = true; + let err = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }) + .add_file("secret.txt", b"classified") + .build() + .unwrap_err(); + assert!(matches!(err, r7z::R7zError::InvalidOptions(_))); + + let mut enc = r7z::EncryptionOptions::default_for_password("Secret123"); + enc.num_cycles_power = 25; + let mut buf = std::io::Cursor::new(Vec::new()); + let err = match r7z::ArchiveWriter::new( + &mut buf, + r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }, + ) { + Ok(_) => panic!("ArchiveWriter accepted unsupported AES cycle power"), + Err(err) => err, + }; + assert!(matches!(err, r7z::R7zError::InvalidOptions(_))); + + let mut enc = r7z::EncryptionOptions::default_for_password("Secret123"); + enc.encrypt_header = true; + let invalid_options = r7z::ArchiveOptions { + header_mode: r7z::HeaderMode::Plain, + encryption: Some(enc), + ..Default::default() + }; + let err = r7z::ArchiveBuilder::new() + .options(invalid_options.clone()) + .add_file("secret.txt", b"classified") + .build() + .unwrap_err(); + assert!(matches!(err, r7z::R7zError::InvalidOptions(_))); + + let mut buf = std::io::Cursor::new(Vec::new()); + let err = match r7z::ArchiveWriter::new(&mut buf, invalid_options) { + Ok(_) => panic!("ArchiveWriter accepted invalid encrypted header options"), + Err(err) => err, + }; + assert!(matches!(err, r7z::R7zError::InvalidOptions(_))); +} + +#[test] +fn archive_builder_aes_encrypted_header_p7zip_and_r7z_require_password() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("aes_header.7z"); + let mut enc = r7z::EncryptionOptions::default_for_password("HeaderSecret"); + enc.encrypt_header = true; + let options = r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }; + let bytes = r7z::ArchiveBuilder::new() + .options(options) + .add_file("hidden.txt", b"hidden payload") + .build() + .expect("build failed"); + std::fs::write(&archive_path, bytes).unwrap(); + + let err = match r7z::Archive::open(&archive_path) { + Ok(_) => panic!("encrypted header opened without password"), + Err(err) => err, + }; + assert!(matches!(err, r7z::R7zError::PasswordRequired)); + let archive = r7z::Archive::open_with_password(&archive_path, Some("HeaderSecret")).unwrap(); + assert_eq!( + archive + .extract_to_memory_with_password(0, Some("HeaderSecret")) + .unwrap(), + b"hidden payload" + ); + + let no_password = run_7z(&["l", archive_path.to_str().unwrap()], dir); + assert!(!no_password.status.success()); + + let out_dir = dir.join("out"); + let out_arg = format!("-o{}", out_dir.to_str().unwrap()); + let out = run_7z( + &[ + "x", + "-y", + "-pHeaderSecret", + archive_path.to_str().unwrap(), + &out_arg, + ], + dir, + ); + assert!( + out.status.success(), + "7z x failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert_eq!( + std::fs::read(out_dir.join("hidden.txt")).unwrap(), + b"hidden payload" + ); +} + +#[test] +fn archive_builder_empty_only_encrypted_header_p7zip_and_r7z_read() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path(); + let archive_path = dir.join("empty_aes_header.7z"); + let mut enc = r7z::EncryptionOptions::default_for_password("HeaderSecret"); + enc.encrypt_header = true; + let options = r7z::ArchiveOptions { + encryption: Some(enc), + ..Default::default() + }; + let bytes = r7z::ArchiveBuilder::new() + .options(options) + .add_directory("emptydir", r7z::EntryMeta::directory_unix_mode(0o040_755)) + .add_empty_file("emptydir/empty.txt", r7z::EntryMeta::archive_file()) + .add_anti_item("removed.txt", r7z::EntryMeta::default()) + .build() + .expect("build failed"); + std::fs::write(&archive_path, bytes).unwrap(); + + let err = match r7z::Archive::open(&archive_path) { + Ok(_) => panic!("encrypted empty header opened without password"), + Err(err) => err, + }; + assert!(matches!(err, r7z::R7zError::PasswordRequired)); + + let archive = r7z::Archive::open_with_password(&archive_path, Some("HeaderSecret")).unwrap(); + assert!(archive.streams_info().is_none()); + let fi = archive.files_info().unwrap(); + assert_eq!(archive.num_files(), 3); + assert!(fi.is_directory(0)); + assert!(fi.is_empty_file(1)); + assert!(fi.is_anti(2)); + assert_eq!( + archive + .extract_to_memory_with_password(1, Some("HeaderSecret")) + .unwrap(), + b"" + ); + + let no_password = run_7z(&["l", archive_path.to_str().unwrap()], dir); + assert!(!no_password.status.success()); + + let listing = run_7z( + &["l", "-pHeaderSecret", archive_path.to_str().unwrap()], + dir, + ); + assert!( + listing.status.success(), + "7z l failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&listing.stdout), + String::from_utf8_lossy(&listing.stderr) + ); + let listing_stdout = String::from_utf8_lossy(&listing.stdout); + assert!(listing_stdout.contains("emptydir/empty.txt")); + assert!(listing_stdout.contains("removed.txt")); + + let out_dir = dir.join("out"); + let out_arg = format!("-o{}", out_dir.to_str().unwrap()); + let out = run_7z( + &[ + "x", + "-y", + "-pHeaderSecret", + archive_path.to_str().unwrap(), + &out_arg, + ], + dir, + ); + assert!( + out.status.success(), + "7z x failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + assert!(out_dir.join("emptydir").is_dir()); + assert_eq!( + std::fs::read(out_dir.join("emptydir/empty.txt")).unwrap(), + b"" + ); +} + +#[test] +fn compression_options_control_lzma2_properties_and_solid_blocks() { + let options = r7z::ArchiveOptions { + compression: r7z::CompressionOptions { + dictionary_size: Some(1 << 20), + solid: r7z::SolidMode::NonSolid, + ..Default::default() + }, + ..Default::default() + }; + let bytes = r7z::ArchiveBuilder::new() + .options(options) + .add_file("a.txt", b"alpha") + .add_file("b.txt", b"bravo") + .build() + .unwrap(); + let archive = r7z::Archive::from_bytes(bytes.into()).unwrap(); + let unpack_info = archive + .streams_info() + .unwrap() + .unpack_info + .as_ref() + .unwrap(); + assert_eq!(unpack_info.num_folders, 2); + let folder = unpack_info.parse_folder(0).unwrap(); + assert_eq!(folder.coders[0].properties.as_deref(), Some(&[16][..])); +} + +#[test] +fn build_streaming_to_writer_matches_seek_backed_output() { + let files = vec![ + ("a.txt".to_string(), b"alpha".as_slice()), + ("b.txt".to_string(), b"bravo".as_slice()), + ]; + let mut seek_backed = std::io::Cursor::new(Vec::new()); + r7z::build_streaming_with_options( + files.clone(), + &mut seek_backed, + r7z::ArchiveOptions::default(), + ) + .unwrap(); + + struct WriteOnly(Vec); + impl std::io::Write for WriteOnly { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let mut write_only = WriteOnly(Vec::new()); + r7z::build_streaming_to_writer(files, &mut write_only, r7z::ArchiveOptions::default()).unwrap(); + assert_eq!(write_only.0, seek_backed.into_inner()); + + let tmp = tempfile::tempdir().unwrap(); + let mut temp_spooled = WriteOnly(Vec::new()); + r7z::build_streaming_to_writer( + vec![ + ("a.txt".to_string(), b"alpha".as_slice()), + ("b.txt".to_string(), b"bravo".as_slice()), + ], + &mut temp_spooled, + r7z::ArchiveOptions { + streaming: r7z::StreamingOptions { + spool: r7z::SpoolMode::TempFile { + dir: Some(tmp.path().to_path_buf()), + }, + ..Default::default() + }, + ..Default::default() + }, + ) + .unwrap(); + let archive = r7z::Archive::from_bytes(temp_spooled.0.into()).unwrap(); + assert_eq!(archive.extract_to_memory(1).unwrap(), b"bravo"); + + let mut auto_spooled = WriteOnly(Vec::new()); + r7z::build_streaming_to_writer( + vec![ + ("a.txt".to_string(), b"alpha".as_slice()), + ("b.txt".to_string(), b"bravo".as_slice()), + ], + &mut auto_spooled, + r7z::ArchiveOptions { + streaming: r7z::StreamingOptions { + spool: r7z::SpoolMode::Auto { + memory_threshold: 1, + dir: Some(tmp.path().to_path_buf()), + }, + ..Default::default() + }, + ..Default::default() + }, + ) + .unwrap(); + let archive = r7z::Archive::from_bytes(auto_spooled.0.into()).unwrap(); + assert_eq!(archive.extract_to_memory(0).unwrap(), b"alpha"); + assert_eq!(std::fs::read_dir(tmp.path()).unwrap().count(), 0); +} + +#[test] +fn build_streaming_volumes_splits_final_archive_bytes() { + let tmp = tempfile::tempdir().unwrap(); + let base = tmp.path().join("split.7z"); + let payload = (0u8..=255).cycle().take(16 * 1024).collect::>(); + let entries = vec![("payload.bin".to_string(), payload.as_slice())]; + let options = r7z::ArchiveOptions { + codec: r7z::Codec::Copy, + ..Default::default() + }; + let paths = r7z::build_streaming_volumes( + entries, + &base, + options, + r7z::VolumeOptions { + sizes: vec![NonZeroU64::new(2048).unwrap()], + }, + ) + .unwrap(); + assert!(paths.len() > 1); + assert_eq!(paths[0], tmp.path().join("split.7z.001")); + + let mut joined = Vec::new(); + for path in &paths { + joined.extend_from_slice(&std::fs::read(path).unwrap()); + } + let archive = r7z::Archive::from_bytes(joined.into()).unwrap(); + assert_eq!(archive.extract_to_memory(0).unwrap(), payload); +} + +#[test] +fn symlink_entries_round_trip_as_metadata_and_regular_extraction() { + let tmp = tempfile::tempdir().unwrap(); + let out = tmp.path().join("out"); + let bytes = r7z::ArchiveBuilder::new() + .add_symlink("link.txt", "target.txt", r7z::EntryMeta::default()) + .build() + .unwrap(); + let archive = r7z::Archive::from_bytes(bytes.into()).unwrap(); + let fi = archive.files_info().unwrap(); + assert!(fi.is_symlink(0)); + assert_eq!(fi.entry_type(0), r7z::EntryType::Symlink); + assert_eq!( + archive.symlink_target(0).unwrap().as_deref(), + Some("target.txt") + ); + + archive.extract_all(&out).unwrap(); + assert_eq!(std::fs::read(out.join("link.txt")).unwrap(), b"target.txt"); + assert!(!std::fs::symlink_metadata(out.join("link.txt")) + .unwrap() + .file_type() + .is_symlink()); +} diff --git a/tests/lzma_perf_test.rs b/tests/lzma_perf_test.rs index 0cd4dfe..481a7d6 100644 --- a/tests/lzma_perf_test.rs +++ b/tests/lzma_perf_test.rs @@ -1,3 +1,5 @@ +#![allow(clippy::pedantic)] + /// Head-to-head comparison: lzma_rust2 (pure Rust) vs C liblzma (via xz2). /// /// Uses LZMA-alone format for both so the underlying algorithm is identical diff --git a/tests/p7zip_extract_parity_test.rs b/tests/p7zip_extract_parity_test.rs index 2db928e..9298d09 100644 --- a/tests/p7zip_extract_parity_test.rs +++ b/tests/p7zip_extract_parity_test.rs @@ -103,6 +103,10 @@ fn streaming_extract_reports_corrupt_lzma_and_lzma2_payloads() { #[test] fn streaming_extract_stops_after_target_when_folder_crc_is_absent() { let mut bytes = r7z::ArchiveBuilder::new() + .options(r7z::ArchiveOptions { + header_mode: r7z::HeaderMode::Plain, + ..Default::default() + }) .compression(r7z::Codec::Lzma2) .add_file("first.txt", b"first") .add_file("second.bin", &vec![0xA5u8; 128 * 1024]) @@ -176,7 +180,9 @@ fn write_with_archive_writer( multi_folder: bool, ) { let file = std::fs::File::create(archive_path).unwrap(); - let mut writer = r7z::ArchiveWriter::new(file).unwrap().compression(codec); + let mut writer = r7z::ArchiveWriter::new(file, r7z::ArchiveOptions::default()) + .unwrap() + .compression(codec); for (idx, (path, data)) in files.iter().enumerate() { if multi_folder && idx == files.len() / 2 { writer.new_folder().unwrap(); diff --git a/tests/read_parity_test.rs b/tests/read_parity_test.rs index adaef3d..1d61bf6 100644 --- a/tests/read_parity_test.rs +++ b/tests/read_parity_test.rs @@ -1,3 +1,5 @@ +#![allow(clippy::pedantic)] + fn build_copy_archive(name: &str, data: &[u8]) -> Vec { let mut header = Vec::new(); header.push(0x01); // Header diff --git a/tests/support/mod.rs b/tests/support/mod.rs index c2f6363..38d456c 100644 --- a/tests/support/mod.rs +++ b/tests/support/mod.rs @@ -1,3 +1,4 @@ +#![allow(clippy::pedantic)] #![allow(dead_code)] use std::{ collections::BTreeSet,