Optimize YMAP and YTYP IO while retaining zlib

fivefury/meta/builder.py

@@ -3,9 +3,9 @@
 import dataclasses
 import struct
 from collections.abc import Iterable, Mapping, Sequence
-from functools import cache
 from typing import Any
 
+from .. import _native_abi3
 from ..binary import align, pad_bytes
 from ..hashing import jenk_hash
 from ..metahash import MetaHash
@@ -24,6 +24,7 @@
     MetaStructInfo,
     RawStruct,
 )
+from .codec import camel_to_snake, scalar_plan
 from .defs import (
     META_TYPE_NAME_ARRAYINFO,
     META_TYPE_NAME_BYTE,
@@ -74,6 +75,7 @@ def __init__(
         self.page_size: int = 0x2000
         self.page_count: int = 1
         self.page_flags: int = 0
+        self._scalar_plans: dict = {}
 
     def register_struct(self, info: MetaStructInfo) -> None:
         if info.name_hash not in self.struct_infos:
@@ -86,6 +88,7 @@ def register_enum(self, info: MetaEnumInfo) -> None:
         self.enum_infos[info.name_hash] = info
 
     def build(self, root_name_hash: int = 0, root_value: Mapping[str, Any] | RawStruct | None = None) -> bytes:
+        self._scalar_plans.clear()
         self.blocks.clear()
         self._block_group.clear()
         self.used_struct_hashes.clear()
@@ -196,11 +199,14 @@ def _encode_struct_payload(self, name_hash: int, value: Mapping[str, Any] | RawS
         if isinstance(value, RawStruct):
             return pad_bytes(value.data, 16)
         struct_info = self._lookup_struct_info(name_hash)
-        payload = bytearray(struct_info.structure_size)
-        for index, entry in enumerate(struct_info.entries):
-            if entry.name_hash == META_TYPE_NAME_ARRAYINFO:
-                continue
-            field_value = self._field_value(value, entry.name)
+        plan = self._scalar_plans.get(name_hash)
+        if plan is None:
+            plan = scalar_plan(struct_info.structure_size, tuple(struct_info.entries))
+            self._scalar_plans[name_hash] = plan
+        capsule, complex_fields = plan
+        payload = _native_abi3.meta_scalars_write(capsule, value, jenk_hash)
+        for index, entry, name in complex_fields:
+            field_value = self._field_value(value, name)
             encoded = self._encode_field(struct_info, index, entry, field_value)
             offset = entry.data_offset
             payload[offset : offset + len(encoded)] = encoded
@@ -209,44 +215,12 @@ def _encode_struct_payload(self, name_hash: int, value: Mapping[str, Any] | RawS
     def _field_value(self, value: Mapping[str, Any], field_name: str) -> Any:
         if field_name in value:
             return value[field_name]
-        snake_name = _camel_to_snake(field_name)
+        snake_name = camel_to_snake(field_name)
         return value.get(snake_name)
 
     def _encode_field(self, struct_info: MetaStructInfo, field_index: int, field: MetaFieldInfo, value: Any) -> bytes:
         data_type = field.data_type
         match data_type:
-            case MetaDataType.BOOLEAN:
-                return struct.pack("<?", bool(value))
-            case MetaDataType.SIGNED_BYTE:
-                return struct.pack("<b", int(value or 0))
-            case MetaDataType.UNSIGNED_BYTE:
-                return struct.pack("<B", int(value or 0))
-            case MetaDataType.SIGNED_SHORT:
-                return struct.pack("<h", int(value or 0))
-            case MetaDataType.UNSIGNED_SHORT:
-                return struct.pack("<H", int(value or 0))
-            case MetaDataType.SIGNED_INT:
-                return struct.pack("<i", int(value or 0))
-            case MetaDataType.UNSIGNED_INT:
-                return struct.pack("<I", int(value or 0))
-            case MetaDataType.FLOAT:
-                return struct.pack("<f", float(value or 0.0))
-            case MetaDataType.FLOAT_XYZ:
-                x, y, z = _coerce_vector(value, 3)
-                return struct.pack("<fff", x, y, z)
-            case MetaDataType.FLOAT_XYZW:
-                x, y, z, w = _coerce_vector(value, 4)
-                return struct.pack("<ffff", x, y, z, w)
-            case MetaDataType.HASH:
-                return struct.pack("<I", _coerce_hash(value))
-            case MetaDataType.BYTE_ENUM:
-                return struct.pack("<B", int(value or 0))
-            case MetaDataType.INT_ENUM:
-                return struct.pack("<i", int(value or 0))
-            case MetaDataType.SHORT_FLAGS:
-                return struct.pack("<h", int(value or 0))
-            case MetaDataType.INT_FLAGS_1 | MetaDataType.INT_FLAGS_2:
-                return struct.pack("<i", int(value or 0))
             case MetaDataType.ARRAY_OF_CHARS:
                 length = field.reference_key & 0xFFFF
                 text = (value or "").encode("ascii", errors="ignore")[:length]
@@ -309,6 +283,8 @@ def _encode_field(self, struct_info: MetaStructInfo, field_index: int, field: Me
                 raise NotImplementedError(f"Unsupported META field type {data_type}")
 
     def _encode_array(self, struct_info: MetaStructInfo, field_index: int, value: Any) -> bytes:
+        if not value:
+            return bytes(16)
         items = list(value or [])
         if field_index <= 0:
             return MetaArrayRef(MetaPointer(0, 0), 0, 0, 0).to_bytes()
@@ -571,16 +547,6 @@ def _value_struct_hash(value: Any, *, fallback: int) -> int:
     raise ValueError("Structure hash is required for pointer/inline structure encoding")
 
 
-@cache
-def _camel_to_snake(value: str) -> str:
-    chars: list[str] = []
-    for index, char in enumerate(value):
-        if char.isupper() and index > 0 and not value[index - 1].isupper():
-            chars.append("_")
-        chars.append(char.lower())
-    return "".join(chars)
-
-
 def _coerce_float_xyz(value: Any) -> tuple[float, float, float]:
     if isinstance(value, Mapping):
         return (

fivefury/meta/codec.py

@@ -0,0 +1,52 @@
+"""Compile immutable META field descriptions for the shared native scalar codec."""
+
+from functools import lru_cache
+
+from .. import _native_abi3
+from .defs import META_TYPE_NAME_ARRAYINFO, MetaDataType
+
+_SCALARS = frozenset(
+    (
+        MetaDataType.BOOLEAN,
+        MetaDataType.SIGNED_BYTE,
+        MetaDataType.UNSIGNED_BYTE,
+        MetaDataType.SIGNED_SHORT,
+        MetaDataType.UNSIGNED_SHORT,
+        MetaDataType.SIGNED_INT,
+        MetaDataType.UNSIGNED_INT,
+        MetaDataType.FLOAT,
+        MetaDataType.FLOAT_XYZ,
+        MetaDataType.FLOAT_XYZW,
+        MetaDataType.HASH,
+        MetaDataType.BYTE_ENUM,
+        MetaDataType.SHORT_FLAGS,
+        MetaDataType.INT_ENUM,
+        MetaDataType.INT_FLAGS_1,
+        MetaDataType.INT_FLAGS_2,
+    )
+)
+
+
+@lru_cache(maxsize=256)
+def camel_to_snake(value: str) -> str:
+    return "".join(
+        ("_" if char.isupper() and index and not value[index - 1].isupper() else "")
+        + char.lower()
+        for index, char in enumerate(value)
+    )
+
+
+@lru_cache(maxsize=256)
+def scalar_plan(size, entries):
+    scalars, complex_fields = [], []
+    for index, entry in enumerate(entries):
+        if entry.name_hash == META_TYPE_NAME_ARRAYINFO:
+            continue
+        name = entry.name
+        if entry.data_type in _SCALARS:
+            scalars.append(
+                (name, camel_to_snake(name), entry.data_offset, int(entry.data_type))
+            )
+        else:
+            complex_fields.append((index, entry, name))
+    return _native_abi3.meta_scalars_new(size, scalars), tuple(complex_fields)

fivefury/meta/read.py

@@ -4,6 +4,7 @@
 import struct
 from typing import Any
 
+from .. import _native_abi3
 from ..binary import BinaryDocument, BinaryScalarType
 from ..resource import parse_rsc7
 from . import (
@@ -19,9 +20,9 @@
     MetaStructInfo,
     RawStruct,
 )
+from .codec import scalar_plan
 from .defs import (
     GRAPHICS_BASE,
-    META_TYPE_NAME_ARRAYINFO,
     STRUCTS_BY_HASH,
     SYSTEM_BASE,
     MetaDataType,
@@ -44,6 +45,7 @@ class ParsedMeta:
     resource_version: int | None = None
     system_flags: int | None = None
     graphics_flags: int | None = None
+    _scalar_plans: dict = dataclasses.field(default_factory=dict, repr=False)
 
     @classmethod
     def from_bytes(cls, data: bytes) -> ParsedMeta:
@@ -127,47 +129,29 @@ def decode_struct(self, name_hash: int, raw: bytes) -> Any:
                 return RawStruct(name_hash, raw[: struct_def.size], None)
             return RawStruct(name_hash, raw, None)
 
-        values: dict[str, Any] = {}
-        for index, entry in enumerate(struct_info.entries):
-            if entry.name_hash == META_TYPE_NAME_ARRAYINFO:
-                continue
-            values[entry.name] = self._decode_field(struct_info, index, entry, raw)
+        plan = self._scalar_plans.get(name_hash)
+        if plan is None:
+            plan = scalar_plan(struct_info.structure_size, tuple(struct_info.entries))
+            self._scalar_plans[name_hash] = plan
+        capsule, complex_fields = plan
+        values = _native_abi3.meta_scalars_read(capsule, raw)
+        for index, entry, name in complex_fields:
+            values[name] = self._decode_field(struct_info, index, entry, raw)
         values["_meta_name_hash"] = name_hash
         values["_meta_name"] = struct_info.name
         return values
 
     def _decode_field(self, struct_info: MetaStructInfo, field_index: int, field: MetaFieldInfo, raw: bytes) -> Any:
         offset = field.data_offset
         data_type = field.data_type
+        if data_type is MetaDataType.ARRAY:
+            pointer, count = struct.unpack_from("<QH", raw, offset)
+            if count == 0 or pointer & 0xFFF == 0:
+                return []
+            return self._resolve_array(
+                struct_info, field_index, MetaArrayRef.from_bytes(raw, offset)
+            )
         match data_type:
-            case MetaDataType.BOOLEAN:
-                return raw[offset] != 0
-            case MetaDataType.SIGNED_BYTE:
-                return struct.unpack_from("<b", raw, offset)[0]
-            case MetaDataType.UNSIGNED_BYTE:
-                return raw[offset]
-            case MetaDataType.SIGNED_SHORT:
-                return struct.unpack_from("<h", raw, offset)[0]
-            case MetaDataType.UNSIGNED_SHORT:
-                return struct.unpack_from("<H", raw, offset)[0]
-            case MetaDataType.SIGNED_INT:
-                return struct.unpack_from("<i", raw, offset)[0]
-            case MetaDataType.UNSIGNED_INT:
-                return struct.unpack_from("<I", raw, offset)[0]
-            case MetaDataType.FLOAT:
-                return struct.unpack_from("<f", raw, offset)[0]
-            case MetaDataType.FLOAT_XYZ:
-                return struct.unpack_from("<fff", raw, offset)
-            case MetaDataType.FLOAT_XYZW:
-                return struct.unpack_from("<ffff", raw, offset)
-            case MetaDataType.HASH:
-                return struct.unpack_from("<I", raw, offset)[0]
-            case MetaDataType.BYTE_ENUM:
-                return raw[offset]
-            case MetaDataType.SHORT_FLAGS:
-                return struct.unpack_from("<h", raw, offset)[0]
-            case MetaDataType.INT_ENUM | MetaDataType.INT_FLAGS_1 | MetaDataType.INT_FLAGS_2:
-                return struct.unpack_from("<i", raw, offset)[0]
             case MetaDataType.ARRAY_OF_CHARS:
                 length = field.reference_key & 0xFFFF
                 end = raw.find(b"\x00", offset, offset + length)
@@ -191,9 +175,6 @@ def _decode_field(self, struct_info: MetaStructInfo, field_index: int, field: Me
                 return self._resolve_struct_pointer(pointer)
             case MetaDataType.STRUCTURE:
                 return self._decode_inline_structure(field.reference_key, raw[offset:])
-            case MetaDataType.ARRAY:
-                array_ref = MetaArrayRef.from_bytes(raw, offset)
-                return self._resolve_array(struct_info, field_index, array_ref)
             case _:
                 return None
 

fivefury/metahash.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
-from typing import Any, ClassVar
+from typing import ClassVar
 
+from . import _native_abi3
 from .hashing import jenk_hash
 
 
@@ -21,20 +22,15 @@ def from_value(cls, value: HashLike | None = 0, *, text: str | None = None) -> M
         return cls(value)
 
     def __init__(self, value: HashLike | None = 0) -> None:
-        match value:
-            case MetaHash():
-                self._value = value.raw
-                self._cached_uint = value._cached_uint
-            case None | "":
-                self._value = 0
-                self._cached_uint = 0
-            case str():
-                self._value = value
-                self._cached_uint = None
-            case _:
-                int_val = int(value)
-                self._value = int_val
-                self._cached_uint = int_val
+        if isinstance(value, MetaHash):
+            raw, cached = value._value, value._cached_uint
+        elif isinstance(value, str):
+            raw, cached = (value, None) if value else (0, 0)
+        else:
+            raw = 0 if value is None else int(value)
+            cached = raw
+        self._value = raw
+        self._cached_uint = cached
 
     @property
     def raw(self) -> int | str:
@@ -133,20 +129,14 @@ def coerce_meta_hash_list(values: Iterable[HashLike] | None) -> list[MetaHash]:
     return [coerce_meta_hash(value) for value in values]
 
 
-class MetaHashFieldsMixin:
+class MetaHashFieldsMixin(_native_abi3.HashFields):
     __slots__ = ()
 
     _hash_fields: ClassVar[tuple[str, ...]] = ()
     _hash_list_fields: ClassVar[tuple[str, ...]] = ()
 
-    def __setattr__(self, name: str, value: Any) -> None:
-        if name in type(self)._hash_fields:
-            object.__setattr__(self, name, coerce_meta_hash(value))
-            return
-        if name in type(self)._hash_list_fields:
-            object.__setattr__(self, name, coerce_meta_hash_list(value))
-            return
-        object.__setattr__(self, name, value)
+    _coerce_hash_field = staticmethod(coerce_meta_hash)
+    _coerce_hash_list_field = staticmethod(coerce_meta_hash_list)
 
 
 __all__ = [

fivefury/resource.py

@@ -387,8 +387,8 @@ def get_resource_flags_from_size_with_page_count(size: int, version: int, page_c
     return best_flags
 
 
-def compress_resource_stream(data: bytes) -> bytes:
-    return zlib.compress(data, level=9, wbits=-15)
+def compress_resource_stream(data: bytes, *, level: int = 9) -> bytes:
+    return zlib.compress(data, level=level, wbits=-15)
 
 
 def _decompress_raw_resource_stream(data: bytes, *, dictionary: bytes | None = None) -> bytes:
@@ -466,8 +466,10 @@ class ResourceSections:
     system_data: bytes
     graphics_data: bytes
 
-    def to_bytes(self) -> bytes:
-        return self.header.pack() + compress_resource_stream(self.system_data + self.graphics_data)
+    def to_bytes(self, *, compression_level: int = 9) -> bytes:
+        return self.header.pack() + compress_resource_stream(
+            self.system_data + self.graphics_data, level=compression_level
+        )
 
 
 @dataclasses.dataclass(frozen=True, slots=True)
@@ -770,6 +772,7 @@ def build_rsc7(
     graphics_alignment: int | None = None,
     system_flags: int | None = None,
     graphics_flags: int | None = None,
+    compression_level: int = 9,
 ) -> bytes:
     return prepare_rsc7_sections(
         system_data,
@@ -779,4 +782,4 @@ def build_rsc7(
         graphics_alignment=graphics_alignment,
         system_flags=system_flags,
         graphics_flags=graphics_flags,
-    ).to_bytes()
+    ).to_bytes(compression_level=compression_level)

fivefury/vector.py

@@ -12,7 +12,8 @@ def lerp(start: float, end: float, amount: float) -> float:
 
 
 def _components(value: Iterable[float], size: int, name: str) -> tuple[float, ...]:
-    result = tuple(float(component) for component in value)
+    # Constructors coerce each component; only materialize and check arity here.
+    result = tuple(value)
     if len(result) != size:
         raise ValueError(f"{name} requires exactly {size} components")
     return result