feat(ltk_modpkg): optimize modpkg chunk processing

crates/ltk_modpkg/src/builder.rs

@@ -232,24 +232,8 @@ impl ModpkgBuilder {
         layer_index_map: &HashMap<u64, u32>,
         wad_indices: &HashMap<u64, u32>,
     ) -> Result<Vec<ModpkgChunk>, ModpkgBuilderError> {
-        // First process all meta chunks (metadata, thumbnail, etc.)
         let mut meta_chunks = self.process_meta_chunks(writer, chunk_path_indices)?;
-
-        // Build a set of path hashes for all meta chunks we just emitted,
-        // so we can skip them when processing the remaining chunks.
-        let meta_path_hashes = meta_chunks
-            .iter()
-            .map(|c| c.path_hash)
-            .collect::<HashSet<_>>();
-
-        // Then process all non-meta chunks via the generic pipeline
-        let regular_chunks: Vec<_> = self
-            .chunks
-            .values()
-            .chain(self.meta_chunks.values())
-            .filter(|chunk| !meta_path_hashes.contains(&chunk.path_hash))
-            .collect();
-
+        let regular_chunks = self.collect_regular_chunks(&meta_chunks);
         let mut processed_regular_chunks = Self::process_chunks(
             &regular_chunks,
             writer,
@@ -264,6 +248,27 @@ impl ModpkgBuilder {
         Ok(meta_chunks)
     }
 
+    /// Collect all non-meta chunks, sorted by WAD name then layer.
+    ///
+    /// This groups related chunks physically in the file,
+    /// enabling more sequential I/O when reading all overrides for a WAD.
+    fn collect_regular_chunks(&self, meta_chunks: &[ModpkgChunk]) -> Vec<&ModpkgChunkBuilder> {
+        let meta_path_hashes = meta_chunks
+            .iter()
+            .map(|c| c.path_hash)
+            .collect::<HashSet<_>>();
+
+        let mut regular_chunks: Vec<_> = self
+            .chunks
+            .values()
+            .chain(self.meta_chunks.values())
+            .filter(|chunk| !meta_path_hashes.contains(&chunk.path_hash))
+            .collect();
+        regular_chunks.sort_by(|a, b| a.wad.cmp(&b.wad).then(a.layer.cmp(&b.layer)));
+
+        regular_chunks
+    }
+
     fn write_chunk_toc<W: io::Write + io::Seek>(
         writer: &mut W,
         chunk_toc_offset: u64,
@@ -586,6 +591,14 @@ impl ModpkgChunkBuilder {
         self
     }
 
+    /// Set the WAD association for this chunk.
+    ///
+    /// This enables efficient WAD-based lookups via the secondary index.
+    pub fn with_wad(mut self, wad: &str) -> Self {
+        self.wad = wad.to_lowercase();
+        self
+    }
+
     pub fn path_hash(&self) -> u64 {
         self.path_hash
     }

crates/ltk_modpkg/src/decoder.rs

@@ -32,13 +32,10 @@ where
     }
 
     fn decode_zstd_chunk(&mut self, chunk: &ModpkgChunk) -> Result<Box<[u8]>, ModpkgError> {
-        self.source.seek(SeekFrom::Start(chunk.data_offset))?;
-
-        let mut data: Vec<u8> = vec![0; chunk.uncompressed_size as usize];
+        let compressed = self.load_chunk_raw(chunk)?;
 
-        zstd::Decoder::new(&mut self.source)
-            .map_err(|e| ModpkgError::Io(std::io::Error::other(e)))?
-            .read_exact(&mut data)?;
+        let data = zstd::bulk::decompress(&compressed, chunk.uncompressed_size as usize)
+            .map_err(|e| ModpkgError::Io(std::io::Error::other(e)))?;
 
         Ok(data.into_boxed_slice())
     }

crates/ltk_modpkg/src/lib.rs

@@ -34,6 +34,9 @@ pub use utils::*;
 /// The name of the base layer.
 pub const BASE_LAYER_NAME: &str = "base";
 
+/// A batch-loaded chunk entry: `(path_hash, layer_hash, decompressed_data)`.
+pub type BatchChunkEntry = (u64, u64, Box<[u8]>);
+
 /// The name of the metadata folder inside the mod package.
 pub const METADATA_FOLDER_NAME: &str = "_meta_";
 
@@ -55,6 +58,11 @@ pub struct Modpkg<TSource: Read + Seek> {
     /// The key is a tuple of the path hash and the layer hash respectively.
     pub chunks: HashMap<(u64, u64), ModpkgChunk>,
 
+    /// Secondary index: chunks grouped by (wad_index, layer_index).
+    ///
+    /// Values are chunk keys `(path_hash, layer_hash)` that can be looked up in `chunks`.
+    pub chunks_by_wad_layer: HashMap<(u32, u32), Vec<(u64, u64)>>,
+
     /// The original byte source.
     source: TSource,
 }
@@ -93,9 +101,27 @@ impl<TSource: Read + Seek> Modpkg<TSource> {
         }
     }
 
-    fn candidate_path_hashes(path: &str) -> (u64, Option<u64>) {
+    /// Resolve the chunk key `(path_hash, layer_hash)` for a given path and layer,
+    /// handling both literal and hex-encoded chunk names.
+    ///
+    /// Returns the first matching key, or `Err` if no chunk matches.
+    fn resolve_chunk_key(
+        &self,
+        path: &str,
+        layer: Option<&str>,
+    ) -> Result<(u64, u64), ModpkgError> {
         let normalized = utils::normalize_chunk_path(path);
         let literal_hash = hash_chunk_name(&normalized);
+        let layer_hash = match layer {
+            Some(name) => hash_layer_name(name),
+            None => NO_LAYER_HASH,
+        };
+
+        if self.chunks.contains_key(&(literal_hash, layer_hash)) {
+            return Ok((literal_hash, layer_hash));
+        }
+
+        // Try hex-encoded chunk name fallback (e.g., "abcdef1234567890.dds")
         let filename_lower = Path::new(&normalized)
             .file_name()
             .and_then(|s| s.to_str())
@@ -105,12 +131,14 @@ impl<TSource: Read + Seek> Modpkg<TSource> {
         if utils::is_hex_chunk_name(&filename_lower) {
             if let Some(base) = filename_lower.split('.').next() {
                 if let Ok(parsed) = u64::from_str_radix(base, 16) {
-                    return (literal_hash, Some(parsed));
+                    if self.chunks.contains_key(&(parsed, layer_hash)) {
+                        return Ok((parsed, layer_hash));
+                    }
                 }
             }
         }
 
-        (literal_hash, None)
+        Err(ModpkgError::MissingChunk(literal_hash))
     }
 
     /// Load the raw data of a chunk using the path hash and layer hash
@@ -145,19 +173,8 @@ impl<TSource: Read + Seek> Modpkg<TSource> {
         path: &str,
         layer: Option<&str>,
     ) -> Result<Box<[u8]>, ModpkgError> {
-        let (literal_hash, hex_hash) = Self::candidate_path_hashes(path);
-        let layer_hash = match layer {
-            Some(layer_name) => hash_layer_name(layer_name),
-            None => NO_LAYER_HASH,
-        };
-
-        if let Ok(data) = self.load_chunk_raw(literal_hash, layer_hash) {
-            return Ok(data);
-        }
-        if let Some(hh) = hex_hash {
-            return self.load_chunk_raw(hh, layer_hash);
-        }
-        self.load_chunk_raw(literal_hash, layer_hash)
+        let (ph, lh) = self.resolve_chunk_key(path, layer)?;
+        self.load_chunk_raw(ph, lh)
     }
 
     /// Load and decompress the data of a chunk by path and layer name
@@ -166,38 +183,14 @@ impl<TSource: Read + Seek> Modpkg<TSource> {
         path: &str,
         layer: Option<&str>,
     ) -> Result<Box<[u8]>, ModpkgError> {
-        let (literal_hash, hex_hash) = Self::candidate_path_hashes(path);
-        let layer_hash = match layer {
-            Some(layer_name) => hash_layer_name(layer_name),
-            None => NO_LAYER_HASH,
-        };
-
-        if let Ok(data) = self.load_chunk_decompressed_by_hash(literal_hash, layer_hash) {
-            return Ok(data);
-        }
-        if let Some(hh) = hex_hash {
-            return self.load_chunk_decompressed_by_hash(hh, layer_hash);
-        }
-        self.load_chunk_decompressed_by_hash(literal_hash, layer_hash)
+        let (ph, lh) = self.resolve_chunk_key(path, layer)?;
+        self.load_chunk_decompressed_by_hash(ph, lh)
     }
 
     /// Get a chunk by path and layer name
     pub fn get_chunk(&self, path: &str, layer: Option<&str>) -> Result<&ModpkgChunk, ModpkgError> {
-        let (literal_hash, hex_hash) = Self::candidate_path_hashes(path);
-        let layer_hash = match layer {
-            Some(layer_name) => hash_layer_name(layer_name),
-            None => NO_LAYER_HASH,
-        };
-
-        if let Some(chunk) = self.chunks.get(&(literal_hash, layer_hash)) {
-            return Ok(chunk);
-        }
-        if let Some(hh) = hex_hash {
-            if let Some(chunk) = self.chunks.get(&(hh, layer_hash)) {
-                return Ok(chunk);
-            }
-        }
-        Err(ModpkgError::MissingChunk(literal_hash))
+        let (ph, lh) = self.resolve_chunk_key(path, layer)?;
+        Ok(self.chunks.get(&(ph, lh)).unwrap())
     }
 
     /// Load a chunk into memory
@@ -209,20 +202,67 @@ impl<TSource: Read + Seek> Modpkg<TSource> {
     }
 
     /// Check if a chunk exists by path and layer name
-    pub fn has_chunk(&self, path: &str, layer: Option<&str>) -> Result<bool, ModpkgError> {
-        let (literal_hash, hex_hash) = Self::candidate_path_hashes(path);
-        let layer_hash = match layer {
-            Some(layer_name) => hash_layer_name(layer_name),
-            None => NO_LAYER_HASH,
-        };
+    pub fn has_chunk(&self, path: &str, layer: Option<&str>) -> bool {
+        self.resolve_chunk_key(path, layer).is_ok()
+    }
 
-        if self.chunks.contains_key(&(literal_hash, layer_hash)) {
-            return Ok(true);
-        }
-        if let Some(hh) = hex_hash {
-            return Ok(self.chunks.contains_key(&(hh, layer_hash)));
+    /// Resolve a layer name to its index in the layer table.
+    pub fn layer_index(&self, layer: &str) -> Option<u32> {
+        let layer_hash = hash_layer_name(layer);
+        self.layer_indices
+            .iter()
+            .position(|&h| h == layer_hash)
+            .map(|idx| idx as u32)
+    }
+
+    /// Resolve a WAD name to its index in the WAD table.
+    pub fn wad_index(&self, wad_name: &str) -> Option<u32> {
+        let wad_hash = hash_wad_name(wad_name);
+        self.wads_indices
+            .iter()
+            .position(|&h| h == wad_hash)
+            .map(|idx| idx as u32)
+    }
+
+    /// Get the WAD name for a given WAD index, or `None` if the index is invalid.
+    pub fn wad_name_for_index(&self, wad_index: u32) -> Option<&str> {
+        let wad_hash = self.wads_indices.get(wad_index as usize)?;
+        self.wads.get(wad_hash).map(|s| s.as_str())
+    }
+
+    /// Get the chunk keys for a given (wad_index, layer_index) pair.
+    ///
+    /// Returns an empty slice if no chunks match.
+    pub fn chunks_for_wad_layer(&self, wad_index: u32, layer_index: u32) -> &[(u64, u64)] {
+        self.chunks_by_wad_layer
+            .get(&(wad_index, layer_index))
+            .map(|v| v.as_slice())
+            .unwrap_or(&[])
+    }
+
+    /// Load and decompress multiple chunks in offset-sorted order for better I/O performance.
+    ///
+    /// Returns `(path_hash, layer_hash, data)` tuples in arbitrary order.
+    pub fn load_chunks_batch(
+        &mut self,
+        keys: &[(u64, u64)],
+    ) -> Result<Vec<BatchChunkEntry>, ModpkgError> {
+        // Resolve keys to chunks and sort by data_offset for sequential I/O
+        let mut sorted: Vec<_> = keys
+            .iter()
+            .filter_map(|&(ph, lh)| self.chunks.get(&(ph, lh)).map(|c| (ph, lh, *c)))
+            .collect();
+        sorted.sort_by_key(|(_, _, c)| c.data_offset);
+
+        let mut results = Vec::with_capacity(sorted.len());
+        let mut decoder = ModpkgDecoder {
+            source: &mut self.source,
+        };
+        for (ph, lh, chunk) in &sorted {
+            let data = decoder.load_chunk_decompressed(chunk)?;
+            results.push((*ph, *lh, data));
         }
-        Ok(false)
+        Ok(results)
     }
 }
 
@@ -394,9 +434,9 @@ mod tests {
         let modpkg = Modpkg::mount_from_reader(cursor).unwrap();
 
         // Test has_chunk
-        assert!(modpkg.has_chunk(path, Some(layer_name)).unwrap());
-        assert!(modpkg.has_chunk(hex_path, Some(layer_name)).unwrap());
-        assert!(!modpkg.has_chunk("nonexistent", Some(layer_name)).unwrap());
+        assert!(modpkg.has_chunk(path, Some(layer_name)));
+        assert!(modpkg.has_chunk(hex_path, Some(layer_name)));
+        assert!(!modpkg.has_chunk("nonexistent", Some(layer_name)));
 
         // Test get_chunk
         let chunk = modpkg.get_chunk(path, Some(layer_name)).unwrap();

crates/ltk_modpkg/src/project.rs

@@ -344,16 +344,49 @@ fn build_chunk_from_file(
         .strip_prefix(layer_dir)
         .map_err(|e| PackError::Io(io::Error::other(e.to_string())))?;
 
-    let chunk_builder = ModpkgChunkBuilder::new()
-        .with_path(relative_path.as_str())
+    let relative_str = relative_path.as_str();
+
+    // Detect WAD association from the directory structure.
+    // If the first path component is a WAD name (e.g., "Aatrox.wad.client"),
+    // strip it from the chunk path and track it via wad_index instead.
+    let (chunk_path, wad_name) = match relative_str.split_once('/') {
+        Some((first, rest)) if first.to_ascii_lowercase().ends_with(".wad.client") => {
+            (rest, Some(first))
+        }
+        _ => (relative_str, None),
+    };
+
+    let compression = compression_for_extension(file_path.extension());
+
+    let mut chunk_builder = ModpkgChunkBuilder::new()
+        .with_path(chunk_path)
         .map_err(PackError::Builder)?
-        .with_compression(ModpkgCompression::Zstd)
+        .with_compression(compression)
         .with_layer(&layer.name);
 
+    if let Some(wad) = wad_name {
+        chunk_builder = chunk_builder.with_wad(wad);
+    }
+
     let path_hash = chunk_builder.path_hash();
     Ok((builder.with_chunk(chunk_builder), path_hash))
 }
 
+/// Determine the best compression strategy based on file extension.
+///
+/// Pre-compressed formats (textures, audio) gain little from zstd and
+/// waste CPU time at both compression and decompression.
+fn compression_for_extension(ext: Option<&str>) -> ModpkgCompression {
+    match ext.map(|e| e.to_ascii_lowercase()).as_deref() {
+        // Pre-compressed textures and images
+        Some("dds" | "tex" | "webp" | "png" | "jpg" | "jpeg") => ModpkgCompression::None,
+        // Pre-compressed audio
+        Some("bnk" | "wpk" | "wem" | "ogg") => ModpkgCompression::None,
+        // Everything else benefits from compression
+        _ => ModpkgCompression::Zstd,
+    }
+}
+
 fn add_meta_chunks(
     mut builder: ModpkgBuilder,
     project_root: &Utf8Path,