diff --git a/fivefury/meta/builder.py b/fivefury/meta/builder.py index 4b3558f..349e2ab 100644 --- a/fivefury/meta/builder.py +++ b/fivefury/meta/builder.py @@ -3,9 +3,9 @@ import dataclasses import struct from collections.abc import Iterable, Mapping, Sequence -from functools import cache from typing import Any +from .. import _native_abi3 from ..binary import align, pad_bytes from ..hashing import jenk_hash from ..metahash import MetaHash @@ -24,6 +24,7 @@ MetaStructInfo, RawStruct, ) +from .codec import camel_to_snake, scalar_plan from .defs import ( META_TYPE_NAME_ARRAYINFO, META_TYPE_NAME_BYTE, @@ -74,6 +75,7 @@ def __init__( self.page_size: int = 0x2000 self.page_count: int = 1 self.page_flags: int = 0 + self._scalar_plans: dict = {} def register_struct(self, info: MetaStructInfo) -> None: if info.name_hash not in self.struct_infos: @@ -86,6 +88,7 @@ def register_enum(self, info: MetaEnumInfo) -> None: self.enum_infos[info.name_hash] = info def build(self, root_name_hash: int = 0, root_value: Mapping[str, Any] | RawStruct | None = None) -> bytes: + self._scalar_plans.clear() self.blocks.clear() self._block_group.clear() self.used_struct_hashes.clear() @@ -196,11 +199,14 @@ def _encode_struct_payload(self, name_hash: int, value: Mapping[str, Any] | RawS if isinstance(value, RawStruct): return pad_bytes(value.data, 16) struct_info = self._lookup_struct_info(name_hash) - payload = bytearray(struct_info.structure_size) - for index, entry in enumerate(struct_info.entries): - if entry.name_hash == META_TYPE_NAME_ARRAYINFO: - continue - field_value = self._field_value(value, entry.name) + plan = self._scalar_plans.get(name_hash) + if plan is None: + plan = scalar_plan(struct_info.structure_size, tuple(struct_info.entries)) + self._scalar_plans[name_hash] = plan + capsule, complex_fields = plan + payload = _native_abi3.meta_scalars_write(capsule, value, jenk_hash) + for index, entry, name in complex_fields: + field_value = self._field_value(value, name) encoded = self._encode_field(struct_info, index, entry, field_value) offset = entry.data_offset payload[offset : offset + len(encoded)] = encoded @@ -209,44 +215,12 @@ def _encode_struct_payload(self, name_hash: int, value: Mapping[str, Any] | RawS def _field_value(self, value: Mapping[str, Any], field_name: str) -> Any: if field_name in value: return value[field_name] - snake_name = _camel_to_snake(field_name) + snake_name = camel_to_snake(field_name) return value.get(snake_name) def _encode_field(self, struct_info: MetaStructInfo, field_index: int, field: MetaFieldInfo, value: Any) -> bytes: data_type = field.data_type match data_type: - case MetaDataType.BOOLEAN: - return struct.pack(" bytes: + if not value: + return bytes(16) items = list(value or []) if field_index <= 0: return MetaArrayRef(MetaPointer(0, 0), 0, 0, 0).to_bytes() @@ -571,16 +547,6 @@ def _value_struct_hash(value: Any, *, fallback: int) -> int: raise ValueError("Structure hash is required for pointer/inline structure encoding") -@cache -def _camel_to_snake(value: str) -> str: - chars: list[str] = [] - for index, char in enumerate(value): - if char.isupper() and index > 0 and not value[index - 1].isupper(): - chars.append("_") - chars.append(char.lower()) - return "".join(chars) - - def _coerce_float_xyz(value: Any) -> tuple[float, float, float]: if isinstance(value, Mapping): return ( diff --git a/fivefury/meta/codec.py b/fivefury/meta/codec.py new file mode 100644 index 0000000..a45c059 --- /dev/null +++ b/fivefury/meta/codec.py @@ -0,0 +1,52 @@ +"""Compile immutable META field descriptions for the shared native scalar codec.""" + +from functools import lru_cache + +from .. import _native_abi3 +from .defs import META_TYPE_NAME_ARRAYINFO, MetaDataType + +_SCALARS = frozenset( + ( + MetaDataType.BOOLEAN, + MetaDataType.SIGNED_BYTE, + MetaDataType.UNSIGNED_BYTE, + MetaDataType.SIGNED_SHORT, + MetaDataType.UNSIGNED_SHORT, + MetaDataType.SIGNED_INT, + MetaDataType.UNSIGNED_INT, + MetaDataType.FLOAT, + MetaDataType.FLOAT_XYZ, + MetaDataType.FLOAT_XYZW, + MetaDataType.HASH, + MetaDataType.BYTE_ENUM, + MetaDataType.SHORT_FLAGS, + MetaDataType.INT_ENUM, + MetaDataType.INT_FLAGS_1, + MetaDataType.INT_FLAGS_2, + ) +) + + +@lru_cache(maxsize=256) +def camel_to_snake(value: str) -> str: + return "".join( + ("_" if char.isupper() and index and not value[index - 1].isupper() else "") + + char.lower() + for index, char in enumerate(value) + ) + + +@lru_cache(maxsize=256) +def scalar_plan(size, entries): + scalars, complex_fields = [], [] + for index, entry in enumerate(entries): + if entry.name_hash == META_TYPE_NAME_ARRAYINFO: + continue + name = entry.name + if entry.data_type in _SCALARS: + scalars.append( + (name, camel_to_snake(name), entry.data_offset, int(entry.data_type)) + ) + else: + complex_fields.append((index, entry, name)) + return _native_abi3.meta_scalars_new(size, scalars), tuple(complex_fields) diff --git a/fivefury/meta/read.py b/fivefury/meta/read.py index 90d695a..65c6e99 100644 --- a/fivefury/meta/read.py +++ b/fivefury/meta/read.py @@ -4,6 +4,7 @@ import struct from typing import Any +from .. import _native_abi3 from ..binary import BinaryDocument, BinaryScalarType from ..resource import parse_rsc7 from . import ( @@ -19,9 +20,9 @@ MetaStructInfo, RawStruct, ) +from .codec import scalar_plan from .defs import ( GRAPHICS_BASE, - META_TYPE_NAME_ARRAYINFO, STRUCTS_BY_HASH, SYSTEM_BASE, MetaDataType, @@ -44,6 +45,7 @@ class ParsedMeta: resource_version: int | None = None system_flags: int | None = None graphics_flags: int | None = None + _scalar_plans: dict = dataclasses.field(default_factory=dict, repr=False) @classmethod def from_bytes(cls, data: bytes) -> ParsedMeta: @@ -127,11 +129,14 @@ def decode_struct(self, name_hash: int, raw: bytes) -> Any: return RawStruct(name_hash, raw[: struct_def.size], None) return RawStruct(name_hash, raw, None) - values: dict[str, Any] = {} - for index, entry in enumerate(struct_info.entries): - if entry.name_hash == META_TYPE_NAME_ARRAYINFO: - continue - values[entry.name] = self._decode_field(struct_info, index, entry, raw) + plan = self._scalar_plans.get(name_hash) + if plan is None: + plan = scalar_plan(struct_info.structure_size, tuple(struct_info.entries)) + self._scalar_plans[name_hash] = plan + capsule, complex_fields = plan + values = _native_abi3.meta_scalars_read(capsule, raw) + for index, entry, name in complex_fields: + values[name] = self._decode_field(struct_info, index, entry, raw) values["_meta_name_hash"] = name_hash values["_meta_name"] = struct_info.name return values @@ -139,35 +144,14 @@ def decode_struct(self, name_hash: int, raw: bytes) -> Any: def _decode_field(self, struct_info: MetaStructInfo, field_index: int, field: MetaFieldInfo, raw: bytes) -> Any: offset = field.data_offset data_type = field.data_type + if data_type is MetaDataType.ARRAY: + pointer, count = struct.unpack_from(" M return cls(value) def __init__(self, value: HashLike | None = 0) -> None: - match value: - case MetaHash(): - self._value = value.raw - self._cached_uint = value._cached_uint - case None | "": - self._value = 0 - self._cached_uint = 0 - case str(): - self._value = value - self._cached_uint = None - case _: - int_val = int(value) - self._value = int_val - self._cached_uint = int_val + if isinstance(value, MetaHash): + raw, cached = value._value, value._cached_uint + elif isinstance(value, str): + raw, cached = (value, None) if value else (0, 0) + else: + raw = 0 if value is None else int(value) + cached = raw + self._value = raw + self._cached_uint = cached @property def raw(self) -> int | str: @@ -133,20 +129,14 @@ def coerce_meta_hash_list(values: Iterable[HashLike] | None) -> list[MetaHash]: return [coerce_meta_hash(value) for value in values] -class MetaHashFieldsMixin: +class MetaHashFieldsMixin(_native_abi3.HashFields): __slots__ = () _hash_fields: ClassVar[tuple[str, ...]] = () _hash_list_fields: ClassVar[tuple[str, ...]] = () - def __setattr__(self, name: str, value: Any) -> None: - if name in type(self)._hash_fields: - object.__setattr__(self, name, coerce_meta_hash(value)) - return - if name in type(self)._hash_list_fields: - object.__setattr__(self, name, coerce_meta_hash_list(value)) - return - object.__setattr__(self, name, value) + _coerce_hash_field = staticmethod(coerce_meta_hash) + _coerce_hash_list_field = staticmethod(coerce_meta_hash_list) __all__ = [ diff --git a/fivefury/resource.py b/fivefury/resource.py index a6c2037..418818b 100644 --- a/fivefury/resource.py +++ b/fivefury/resource.py @@ -387,8 +387,8 @@ def get_resource_flags_from_size_with_page_count(size: int, version: int, page_c return best_flags -def compress_resource_stream(data: bytes) -> bytes: - return zlib.compress(data, level=9, wbits=-15) +def compress_resource_stream(data: bytes, *, level: int = 9) -> bytes: + return zlib.compress(data, level=level, wbits=-15) def _decompress_raw_resource_stream(data: bytes, *, dictionary: bytes | None = None) -> bytes: @@ -466,8 +466,10 @@ class ResourceSections: system_data: bytes graphics_data: bytes - def to_bytes(self) -> bytes: - return self.header.pack() + compress_resource_stream(self.system_data + self.graphics_data) + def to_bytes(self, *, compression_level: int = 9) -> bytes: + return self.header.pack() + compress_resource_stream( + self.system_data + self.graphics_data, level=compression_level + ) @dataclasses.dataclass(frozen=True, slots=True) @@ -770,6 +772,7 @@ def build_rsc7( graphics_alignment: int | None = None, system_flags: int | None = None, graphics_flags: int | None = None, + compression_level: int = 9, ) -> bytes: return prepare_rsc7_sections( system_data, @@ -779,4 +782,4 @@ def build_rsc7( graphics_alignment=graphics_alignment, system_flags=system_flags, graphics_flags=graphics_flags, - ).to_bytes() + ).to_bytes(compression_level=compression_level) diff --git a/fivefury/vector.py b/fivefury/vector.py index 4dc569a..621e43b 100644 --- a/fivefury/vector.py +++ b/fivefury/vector.py @@ -12,7 +12,8 @@ def lerp(start: float, end: float, amount: float) -> float: def _components(value: Iterable[float], size: int, name: str) -> tuple[float, ...]: - result = tuple(float(component) for component in value) + # Constructors coerce each component; only materialize and check arity here. + result = tuple(value) if len(result) != size: raise ValueError(f"{name} requires exactly {size} components") return result diff --git a/fivefury/ymap/entities.py b/fivefury/ymap/entities.py index 98cbc2f..db0ecb5 100644 --- a/fivefury/ymap/entities.py +++ b/fivefury/ymap/entities.py @@ -137,7 +137,7 @@ def world_bounds( def from_meta(cls, value: Any) -> EntityDef: return cls( archetype_name=value.get("archetypeName", 0), - flags=coerce_ymap_entity_flags(int(value.get("flags", 0))), + flags=value.get("flags", 0), guid=int(value.get("guid", 0)), position=Vector3.from_iterable(value.get("position", (0.0, 0.0, 0.0))), rotation=Quaternion.from_iterable( @@ -148,11 +148,9 @@ def from_meta(cls, value: Any) -> EntityDef: parent_index=int(value.get("parentIndex", -1)), lod_dist=float(value.get("lodDist", 0.0)), child_lod_dist=float(value.get("childLodDist", 0.0)), - lod_level=coerce_ymap_lod_level(int(value.get("lodLevel", 0))), + lod_level=value.get("lodLevel", 0), num_children=int(value.get("numChildren", 0)), - priority_level=coerce_ymap_priority_level( - int(value.get("priorityLevel", 0)) - ), + priority_level=value.get("priorityLevel", 0), extensions=extensions_from_meta(value.get("extensions", []) or []), ambient_occlusion_multiplier=int( value.get("ambientOcclusionMultiplier", 255) diff --git a/fivefury/ymap/model.py b/fivefury/ymap/model.py index ff4186c..16f7051 100644 --- a/fivefury/ymap/model.py +++ b/fivefury/ymap/model.py @@ -500,7 +500,7 @@ def to_bytes( builder = MetaBuilder(struct_infos=YMAP_STRUCT_INFOS, enum_infos=YMAP_ENUM_INFOS, name=self.meta_name or "") system = builder.build(root_name_hash=meta_name("CMapData"), root_value=self.to_meta_root()) system_flags = builder.page_flags | (((version >> 4) & 0xF) << 28) - return build_rsc7(system, version=version, system_alignment=0x2000, system_flags=system_flags) + return build_rsc7(system, version=version, system_alignment=0x2000, system_flags=system_flags, compression_level=6) def save( self, diff --git a/fivefury/ytyp/base_archetype.py b/fivefury/ytyp/base_archetype.py index 0d74006..c2de367 100644 --- a/fivefury/ytyp/base_archetype.py +++ b/fivefury/ytyp/base_archetype.py @@ -95,7 +95,7 @@ def from_meta(cls, value: Any) -> BaseArchetypeDef: clip_dictionary=value.get("clipDictionary", 0), drawable_dictionary=value.get("drawableDictionary", 0), physics_dictionary=value.get("physicsDictionary", 0), - asset_type=coerce_archetype_asset_type(value.get("assetType", ArchetypeAssetType.UNINITIALIZED)), + asset_type=value.get("assetType", ArchetypeAssetType.UNINITIALIZED), asset_name=value.get("assetName", 0), extensions=extensions_from_meta(value.get("extensions", []) or []), ) diff --git a/fivefury/ytyp/model.py b/fivefury/ytyp/model.py index 8c6e9c1..018cf17 100644 --- a/fivefury/ytyp/model.py +++ b/fivefury/ytyp/model.py @@ -184,7 +184,7 @@ def to_bytes(self, *, version: int = 2, validate: bool = True) -> bytes: builder = MetaBuilder(struct_infos=_ALL_STRUCT_INFOS, enum_infos=YTYP_ENUM_INFOS, name=self.meta_name or "") system = builder.build(root_name_hash=meta_name("CMapTypes"), root_value=self.to_meta_root()) system_flags = builder.page_flags | (((version >> 4) & 0xF) << 28) - return build_rsc7(system, version=version, system_alignment=0x2000, system_flags=system_flags) + return build_rsc7(system, version=version, system_alignment=0x2000, system_flags=system_flags, compression_level=6) def save(self, path: str | Path | None = None, *, version: int = 2, validate: bool = True) -> Path: destination = Path(path) if path is not None else Path(self.suggested_path()) diff --git a/native/python/bridge.h b/native/python/bridge.h index 78787bd..9154d20 100644 --- a/native/python/bridge.h +++ b/native/python/bridge.h @@ -22,6 +22,7 @@ class TextureIndex; } namespace fivefury_py { +PyObject* create_hash_fields_type(); struct BytesView { char* data = nullptr; diff --git a/native/python/hash_fields.cpp b/native/python/hash_fields.cpp new file mode 100644 index 0000000..f21ef0c --- /dev/null +++ b/native/python/hash_fields.cpp @@ -0,0 +1,41 @@ +#include "python/bridge.h" + +namespace fivefury_py { +namespace { +// All other attributes go directly through the normal descriptor protocol. +// Keeping this in tp_setattro avoids a Python frame for every numeric field +// assigned by a dataclass constructor, while preserving hash coercion on edits. +int set_hash_field(PyObject* self, PyObject* name, PyObject* value) { + if (value == nullptr) return PyObject_GenericSetAttr(self, name, nullptr); + auto* cls = reinterpret_cast(Py_TYPE(self)); + constexpr const char* fields[] = {"_hash_fields", "_hash_list_fields"}; + constexpr const char* coercers[] = {"_coerce_hash_field", "_coerce_hash_list_field"}; + for (int i = 0; i < 2; ++i) { + PyHandle names(PyObject_GetAttrString(cls, fields[i])); + if (!names) return -1; + const int contains = PySequence_Contains(names.get(), name); + if (contains < 0) return -1; + if (contains == 0) continue; + PyHandle coerce(PyObject_GetAttrString(cls, coercers[i])); + if (!coerce) return -1; + PyHandle converted(PyObject_CallFunctionObjArgs(coerce.get(), value, nullptr)); + if (!converted) return -1; + return PyObject_GenericSetAttr(self, name, converted.get()); + } + return PyObject_GenericSetAttr(self, name, value); +} +} + +PyObject* create_hash_fields_type() { + PyType_Slot slots[] = { + {Py_tp_setattro, reinterpret_cast(set_hash_field)}, + {Py_tp_new, reinterpret_cast(PyType_GenericNew)}, + {0, nullptr}, + }; + PyType_Spec spec = { + "fivefury._native_abi3.HashFields", sizeof(PyObject), 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, slots, + }; + return PyType_FromSpec(&spec); +} +} // namespace fivefury_py diff --git a/native/python/module.cpp b/native/python/module.cpp index 36d3c45..5e6635d 100644 --- a/native/python/module.cpp +++ b/native/python/module.cpp @@ -11,6 +11,9 @@ namespace fivefury_py { PyMethodDef module_methods[] = { + {"meta_scalars_new", guarded_call, METH_VARARGS, nullptr}, + {"meta_scalars_read", guarded_call, METH_VARARGS, nullptr}, + {"meta_scalars_write", guarded_call, METH_VARARGS, nullptr}, {"index_new", guarded_call, METH_NOARGS, nullptr}, {"index_clear", guarded_call, METH_VARARGS, nullptr}, {"index_count", guarded_call, METH_VARARGS, nullptr}, @@ -127,5 +130,9 @@ PyModuleDef module_def = { } // namespace fivefury_py PyMODINIT_FUNC PyInit__native_abi3(void) { - return PyModule_Create(&fivefury_py::module_def); + fivefury_py::PyHandle module(PyModule_Create(&fivefury_py::module_def)); + if (!module) return nullptr; + fivefury_py::PyHandle hash_fields(fivefury_py::create_hash_fields_type()); + if (!hash_fields || PyModule_AddObjectRef(module.get(), "HashFields", hash_fields.get()) < 0) return nullptr; + return module.release(); } diff --git a/native/resource/bindings.h b/native/resource/bindings.h index d9d7ac4..046a3ff 100644 --- a/native/resource/bindings.h +++ b/native/resource/bindings.h @@ -3,6 +3,9 @@ #include "python/bridge.h" namespace fivefury_py { +PyObject* mod_meta_scalars_new(PyObject*, PyObject*); +PyObject* mod_meta_scalars_read(PyObject*, PyObject*); +PyObject* mod_meta_scalars_write(PyObject*, PyObject*); PyObject* mod_binary_document_new(PyObject*, PyObject* args); PyObject* mod_binary_document_size(PyObject*, PyObject* args); diff --git a/native/resource/meta_codec.cpp b/native/resource/meta_codec.cpp new file mode 100644 index 0000000..935d877 --- /dev/null +++ b/native/resource/meta_codec.cpp @@ -0,0 +1,177 @@ +#include "resource/bindings.h" +#include +#include +#include + +namespace fivefury_py { +namespace { +namespace binary = fivefury_native::binary; +constexpr auto capsule_name = "fivefury.MetaScalars"; +struct Field { + PyHandle name, alias; + int kind; + Py_ssize_t offset; +}; +struct Schema { + Py_ssize_t size; + std::vector fields; +}; +std::size_t width(int kind) { + switch (kind) { + case 1: case 0x10: case 0x11: case 0x60: return 1; + case 0x12: case 0x13: case 0x64: return 2; + case 0x14: case 0x15: case 0x21: case 0x4a: + case 0x62: case 0x63: case 0x65: return 4; + case 0x33: return 12; + case 0x34: return 16; + default: throw std::invalid_argument("Unsupported META scalar type"); + } +} +void destroy_schema(PyObject* capsule) { + delete static_cast(PyCapsule_GetPointer(capsule, capsule_name)); +} +template bool write_integer(char* out, PyObject* value) { + PyHandle integer(PyNumber_Long(value)); + if (!integer) return false; + const auto number = PyLong_AsLongLong(integer.get()); + if (PyErr_Occurred()) return false; + if (number < static_cast(std::numeric_limits::min()) || + number > static_cast(std::numeric_limits::max())) { + PyErr_SetString(PyExc_OverflowError, "META integer exceeds field range"); + return false; + } + binary::store(out, static_cast(number)); + return true; +} +bool write_float(char* out, PyObject* value) { + PyHandle converted(PyNumber_Float(value)); + if (!converted) return false; + const double number = PyFloat_AsDouble(converted.get()); + if (PyErr_Occurred()) return false; + const auto packed = static_cast(number); + if (std::isfinite(number) && !std::isfinite(packed)) { + PyErr_SetString(PyExc_OverflowError, "META float exceeds float32 range"); + return false; + } + binary::store(out, packed); + return true; +} +bool encode(char* out, int kind, PyObject* value, PyObject* hash) { + if (value == Py_None) return true; // Payload is initialized to zero. + if (kind != 0x4a && kind != 0x33 && kind != 0x34) { + const int truth = PyObject_IsTrue(value); + if (truth < 0) return false; + if (!truth) return true; + } + switch (kind) { + case 1: { + int truth = PyObject_IsTrue(value); + if (truth < 0) return false; + *out = static_cast(truth); return true; + } + case 0x10: return write_integer(out, value); + case 0x11: case 0x60: return write_integer(out, value); + case 0x12: case 0x64: return write_integer(out, value); + case 0x13: return write_integer(out, value); + case 0x14: case 0x62: case 0x63: case 0x65: return write_integer(out, value); + case 0x15: return write_integer(out, value); + case 0x4a: { + if (!PyUnicode_Check(value)) return write_integer(out, value); + PyHandle number(PyObject_CallFunctionObjArgs(hash, value, nullptr)); + return number && write_integer(out, number.get()); + } + case 0x21: return write_float(out, value); + default: { + if (PyUnicode_Check(value) || PyBytes_Check(value) || PyByteArray_Check(value)) { + PyErr_SetString(PyExc_TypeError, "Vector values must be numeric iterables"); return false; + } + PyHandle components(PySequence_Tuple(value)); + if (!components) return false; + const Py_ssize_t count = kind == 0x33 ? 3 : 4; + if (PyTuple_Size(components.get()) != count) { + PyErr_SetString(PyExc_ValueError, "Incorrect META vector dimension"); return false; + } + for (Py_ssize_t i = 0; i < count; ++i) + if (!write_float(out + i * 4, PyTuple_GetItem(components.get(), i))) return false; + return true; + } + } +} +PyObject* decode(const char* in, int kind) { + switch (kind) { + case 1: return PyBool_FromLong(*in != 0); + case 0x10: return PyLong_FromLong(binary::load(in)); + case 0x11: case 0x60: return PyLong_FromLong(binary::load(in)); + case 0x12: case 0x64: return PyLong_FromLong(binary::load(in)); + case 0x13: return PyLong_FromLong(binary::load(in)); + case 0x14: case 0x62: case 0x63: case 0x65: return PyLong_FromLong(binary::load(in)); + case 0x15: case 0x4a: return PyLong_FromUnsignedLong(binary::load(in)); + case 0x21: return PyFloat_FromDouble(binary::load(in)); + default: { + const Py_ssize_t count = kind == 0x33 ? 3 : 4; + PyHandle result(PyTuple_New(count)); + if (!result) return nullptr; + for (Py_ssize_t i = 0; i < count; ++i) + if (!tuple_take(result.get(), i, PyFloat_FromDouble(binary::load(in + i * 4)))) return nullptr; + return result.release(); + } + } +} +} + +PyObject* mod_meta_scalars_new(PyObject*, PyObject* args) { + Py_ssize_t size; PyObject* fields; + if (!PyArg_ParseTuple(args, "nO", &size, &fields)) return nullptr; + if (size < 0) throw std::invalid_argument("Negative META structure size"); + auto schema = std::make_unique(); schema->size = size; + PyHandle sequence(PySequence_Tuple(fields)); + if (!sequence) return nullptr; + for (Py_ssize_t i = 0; i < PyTuple_Size(sequence.get()); ++i) { + PyObject *name, *alias; int kind; Py_ssize_t offset; + if (!PyArg_ParseTuple(PyTuple_GetItem(sequence.get(), i), "UUni", &name, &alias, &offset, &kind)) return nullptr; + if (offset < 0 || !binary::contains(offset, width(kind), size)) + throw std::invalid_argument("META field exceeds structure size"); + Py_INCREF(name); Py_INCREF(alias); + schema->fields.push_back({PyHandle(name), PyHandle(alias), kind, offset}); + } + return owned_capsule(std::move(schema), capsule_name, destroy_schema); +} +PyObject* mod_meta_scalars_read(PyObject*, PyObject* args) { + PyObject *capsule, *raw; + if (!PyArg_ParseTuple(args, "OO", &capsule, &raw)) return nullptr; + auto* schema = static_cast(PyCapsule_GetPointer(capsule, capsule_name)); + if (!schema) return nullptr; + Buffer data; if (!data.acquire(raw)) return nullptr; + if (data.len < schema->size) throw std::invalid_argument("META structure is truncated"); + PyHandle result(PyDict_New()); if (!result) return nullptr; + for (const auto& field : schema->fields) { + PyHandle value(decode(static_cast(data.buf) + field.offset, field.kind)); + if (!value || PyDict_SetItem(result.get(), field.name.get(), value.get()) < 0) return nullptr; + } + return result.release(); +} +PyObject* mod_meta_scalars_write(PyObject*, PyObject* args) { + PyObject *capsule, *values, *hash; + if (!PyArg_ParseTuple(args, "OOO", &capsule, &values, &hash)) return nullptr; + auto* schema = static_cast(PyCapsule_GetPointer(capsule, capsule_name)); + if (!schema) return nullptr; + PyHandle result(PyByteArray_FromStringAndSize(nullptr, schema->size)); + if (!result) return nullptr; + auto* out = PyByteArray_AsString(result.get()); + std::memset(out, 0, schema->size); + for (const auto& field : schema->fields) { + PyHandle value(PyObject_GetItem(values, field.name.get())); + if (!value) { + if (!PyErr_ExceptionMatches(PyExc_KeyError)) return nullptr; + PyErr_Clear(); + PyHandle alias(PyObject_GetItem(values, field.alias.get())); + if (!alias) { + if (!PyErr_ExceptionMatches(PyExc_KeyError)) return nullptr; + PyErr_Clear(); continue; + } + if (!encode(out + field.offset, field.kind, alias.get(), hash)) return nullptr; + } else if (!encode(out + field.offset, field.kind, value.get(), hash)) return nullptr; + } + return result.release(); +} +} // namespace fivefury_py diff --git a/setup.py b/setup.py index 8eca562..bcc640e 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ def finalize_options(self) -> None: NATIVE_SOURCE_GROUPS = { - "python": ("bridge.cpp", "module.cpp"), + "python": ("bridge.cpp", "module.cpp", "hash_fields.cpp"), "audio": ("bindings.cpp",), "animation": ( "ycd_bindings.cpp", @@ -44,7 +44,7 @@ def finalize_options(self) -> None: "texture_bindings.cpp", "texture_index.cpp", ), - "resource": ("binary_document.cpp", "bindings.cpp", "layout.cpp"), + "resource": ("binary_document.cpp", "bindings.cpp", "layout.cpp", "meta_codec.cpp"), "rpf": ("archive.cpp", "bindings.cpp", "crypto.cpp", "read.cpp", "read_session.cpp", "scan.cpp"), "spatial": ("bindings.cpp",), } diff --git a/tests/core/serialization/test_resource_stream.py b/tests/core/serialization/test_resource_stream.py index 510b2dd..080ad53 100644 --- a/tests/core/serialization/test_resource_stream.py +++ b/tests/core/serialization/test_resource_stream.py @@ -9,6 +9,7 @@ RSC7_MAGIC, ResourceBlockSpan, ResourceWriter, + build_rsc7, decompress_resource_stream, get_resource_flags_from_size, layout_resource_sections, @@ -18,6 +19,17 @@ ) +def test_compression_level_preserves_header_and_uncompressed_sections(): + system = bytes(range(256)) * 100 + graphics = b"texture" * 1000 + default = build_rsc7(system, graphics_data=graphics) + compact = build_rsc7(system, graphics_data=graphics, compression_level=9) + balanced = build_rsc7(system, graphics_data=graphics, compression_level=6) + assert default == compact + assert parse_rsc7(balanced) == parse_rsc7(compact) + assert zlib.decompress(balanced[16:], wbits=-15) == parse_rsc7(compact)[1] + + @pytest.mark.parametrize("version", [5, 13, 165]) def test_prepare_resource_sections_preserves_padding_and_explicit_flags(version): sections = prepare_rsc7_sections( diff --git a/tests/runtime/test_hash_fields.py b/tests/runtime/test_hash_fields.py new file mode 100644 index 0000000..61a7bcc --- /dev/null +++ b/tests/runtime/test_hash_fields.py @@ -0,0 +1,64 @@ +import copy +import dataclasses +import pickle + +import pytest + +from fivefury.metahash import MetaHash, MetaHashFieldsMixin +from fivefury.ymap import Entity + + +@dataclasses.dataclass(slots=True) +class HashRecord(MetaHashFieldsMixin): + _hash_fields = ('name',) + _hash_list_fields = ('names',) + name: object = 0 + names: list = dataclasses.field(default_factory=list) + count: int = 0 + + +@dataclasses.dataclass(slots=True) +class ChildRecord(HashRecord): + extra: int = 0 + + +def test_inherited_hash_fields_preserve_assignment_and_deletion(): + record = ChildRecord(name='initial', names=['a', 17], count=4, extra=9) + assert isinstance(record.name, MetaHash) + assert all(isinstance(name, MetaHash) for name in record.names) + record.name = 42 + record.names = (name for name in [11, 'b']) + record.count = 7 + assert record.name.raw == 42 + assert [name.raw for name in record.names] == [11, 'b'] + assert record.count == 7 and record.extra == 9 + del record.name + with pytest.raises(AttributeError): + _ = record.name + with pytest.raises(AttributeError): + record.unknown = 0 + + +def test_hash_fields_copy_and_pickle_preserve_nominal_types(): + entity = Entity(archetype_name='prop_test') + for restored in (copy.copy(entity), copy.deepcopy(entity), pickle.loads(pickle.dumps(entity))): + assert restored == entity + assert isinstance(restored.archetype_name, MetaHash) + restored.archetype_name = 17 + assert restored.archetype_name.raw == 17 + assert entity.archetype_name.raw == 'prop_test' + + +def test_native_setter_obeys_descriptors(): + class DescriptorRecord(HashRecord): + @property + def doubled(self): + return self.count * 2 + + @doubled.setter + def doubled(self, value): + self.count = value // 2 + + record = DescriptorRecord() + record.doubled = 12 + assert record.count == 6 diff --git a/tests/runtime/test_meta_scalar_codec.py b/tests/runtime/test_meta_scalar_codec.py new file mode 100644 index 0000000..984ad93 --- /dev/null +++ b/tests/runtime/test_meta_scalar_codec.py @@ -0,0 +1,89 @@ +import struct + +import pytest + +from fivefury import Quaternion, Vector3 +from fivefury import _native_abi3 as native +from fivefury.hashing import jenk_hash +from fivefury.metahash import MetaHash + + +@pytest.mark.parametrize( + "kind,fmt,value", + [ + (1, "?", True), + (0x10, "b", -128), + (0x11, "B", 255), + (0x12, "h", -32768), + (0x13, "H", 65535), + (0x14, "i", -2147483648), + (0x15, "I", 4294967295), + (0x21, "f", 1.25), + (0x60, "B", 255), + (0x64, "h", -32768), + (0x62, "i", -7), + (0x63, "i", -9), + (0x65, "i", 17), + (0x4A, "I", MetaHash("prop_test")), + ], +) +def test_scalar_layout_and_alias(kind, fmt, value): + size = struct.calcsize("<" + fmt) + plan = native.meta_scalars_new(size + 2, [("testValue", "test_value", 1, kind)]) + expected = int(value) if isinstance(value, MetaHash) else value + raw = native.meta_scalars_write(plan, {"test_value": value}, jenk_hash) + assert raw == b"\0" + struct.pack("<" + fmt, expected) + b"\0" + assert native.meta_scalars_read(plan, memoryview(raw)) == {"testValue": expected} + assert native.meta_scalars_write(plan, {}, jenk_hash) == bytes(size + 2) + + +def test_vectors_hashes_and_numeric_conversion(): + plan = native.meta_scalars_new( + 36, + [ + ("position", "position", 0, 0x33), + ("rotation", "rotation", 12, 0x34), + ("name", "name", 28, 0x4A), + ("distance", "distance", 32, 0x21), + ], + ) + value = { + "position": Vector3(1, 2, 3), + "rotation": Quaternion(), + "name": "prop_test", + "distance": "1.5", + } + raw = native.meta_scalars_write(plan, value, jenk_hash) + assert raw == struct.pack("<7fIf", 1, 2, 3, 0, 0, 0, 1, jenk_hash("prop_test"), 1.5) + assert native.meta_scalars_read(plan, raw)["position"] == (1, 2, 3) + with pytest.raises(ValueError): + native.meta_scalars_write(plan, {"position": (1, 2)}, jenk_hash) + with pytest.raises(TypeError): + native.meta_scalars_write(plan, {"position": "123"}, jenk_hash) + + +def test_invalid_layout_buffers_and_overflow(): + with pytest.raises(ValueError): + native.meta_scalars_new(1, [("value", "value", 0, 0x15)]) + with pytest.raises(ValueError): + native.meta_scalars_new(4, [("value", "value", -1, 0x15)]) + with pytest.raises(ValueError): + native.meta_scalars_new(4, [("value", "value", 0, 0xFF)]) + plan = native.meta_scalars_new(4, [("value", "value", 0, 0x15)]) + with pytest.raises(ValueError): + native.meta_scalars_read(plan, b"\0") + with pytest.raises(TypeError): + native.meta_scalars_read(plan, "abcd") + for value in (-1, 2**32): + with pytest.raises(OverflowError): + native.meta_scalars_write(plan, {"value": value}, jenk_hash) + + +def test_scalar_false_values_and_vector_negative_zero(): + plan = native.meta_scalars_new( + 16, [("scalar", "scalar", 0, 0x21), ("vector", "vector", 4, 0x33)] + ) + raw = native.meta_scalars_write( + plan, {"scalar": -0.0, "vector": (-0.0, 0, 0)}, jenk_hash + ) + assert raw == struct.pack("<4f", 0, -0.0, 0, 0)