From b53f2e550d5be3e9b9d39b55a7ceefde9d26115c Mon Sep 17 00:00:00 2001 From: MrVicho13 Date: Wed, 9 Sep 2026 05:51:32 -0300 Subject: [PATCH 1/2] Keep local development documentation outside version control --- .gitignore | 1 + README.md | 2 +- docs/ASSET_OVERLAYS.md | 98 ----------- docs/ASSET_SOURCES.md | 43 ----- docs/AUTHORING_OPERATIONS.md | 100 ----------- docs/AWC_PLAYBACK_CONTRACTS.md | 51 ------ docs/STYLE_GUIDE.md | 292 --------------------------------- docs/YCD_CHANNEL_ENCODING.md | 137 ---------------- docs/YCD_PLAYBACK.md | 77 --------- docs/YED_PLAYBACK.md | 45 ----- docs/YED_RUNTIME_CONTRACTS.md | 211 ------------------------ docs/YMAP_PLACEMENTS.md | 57 ------- 12 files changed, 2 insertions(+), 1112 deletions(-) delete mode 100644 docs/ASSET_OVERLAYS.md delete mode 100644 docs/ASSET_SOURCES.md delete mode 100644 docs/AUTHORING_OPERATIONS.md delete mode 100644 docs/AWC_PLAYBACK_CONTRACTS.md delete mode 100644 docs/STYLE_GUIDE.md delete mode 100644 docs/YCD_CHANNEL_ENCODING.md delete mode 100644 docs/YCD_PLAYBACK.md delete mode 100644 docs/YED_PLAYBACK.md delete mode 100644 docs/YED_RUNTIME_CONTRACTS.md delete mode 100644 docs/YMAP_PLACEMENTS.md diff --git a/.gitignore b/.gitignore index c089d9f..1580dea 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ release_dist_*/ *.egg-info/ references/ +docs/ examples/ .claude/ fivefury-*/ diff --git a/README.md b/README.md index b47eb74..8cf312b 100644 --- a/README.md +++ b/README.md @@ -234,7 +234,7 @@ FiveFury keeps the authoring layer close to the data model: - Stable game-side values use enums for targets, shaders, LODs, flags, render masks, materials, and track formats. - Core writers use atomic replacement and reject known invalid references, ownership, pointers, or packed ranges before replacing the destination. -The full naming, module-boundary, compatibility, performance, and review rules are normative in [`docs/STYLE_GUIDE.md`](docs/STYLE_GUIDE.md). +Local development documentation is kept in `docs/`, which is excluded from version control. ## Scope and guarantees diff --git a/docs/ASSET_OVERLAYS.md b/docs/ASSET_OVERLAYS.md deleted file mode 100644 index abe3a2f..0000000 --- a/docs/ASSET_OVERLAYS.md +++ /dev/null @@ -1,98 +0,0 @@ -# Loose asset overlays - -`GameFileOverlay` resolves a scoped export before consulting an existing -`GameFileCache`. It reads loose files and nested RPFs without copying the -installation index or rescanning the game directory. - -```python -from fivefury import ( - AssetRegistration, - DlcDataFileType, - GameFileOverlay, - GameTarget, -) - -registrations = [ - AssetRegistration( - "common/data/peds/custom_actor.meta", - DlcDataFileType.PED_METADATA, - ), -] - -with GameFileOverlay( - "export/loose", - fallback=installation_cache, # Already scanned; optional. - registrations=registrations, - game=GameTarget.GTA5_ENHANCED, -) as assets: - bundle = assets.resolve_cutscene("scene.cut") - for binding in bundle.bindings.values(): - if binding.ped_init_data is not None: - print(binding.ped_init_data.name) - print(binding.ped_metadata_asset.path) - print(binding.expression_dictionary) - for issue in bundle.diagnostics: - print(issue.code, issue.message) -``` - -## Registrations - -An inventory path alone is not a content registration. Persist each registration's -`path` and `file_type.value` alongside an export and reconstruct `AssetRegistration` -when mounting it. Paths are relative to the loose root; archive-entry paths may -include nested `.rpf` components. - -| Registration | Decoded content | -| --- | --- | -| `DlcDataFileType.PED_METADATA` | `YmtPedMetadata`, including exact model/expression references | -| `DlcDataFileType.EXPRESSION_SETS` | `PedExpressionSetMetadata` | - -These are the supported explicit metadata registrations. RPFs and ordinary -resource files are indexed normally and need no metadata registration here. -The conventional `peds.meta` filename is recognized automatically. Arbitrary -`.meta` filenames are not guessed from their name or XML content. For direct -reading, `read_ped_metadata(source)` accepts XML or binary ped-init YMT. -A component YMT is not ped-init metadata. - -For a standalone cache, `cache.register_metadata(registration)` applies the type -to an already indexed file. Registration requires that the path exists in that -cache and invalidates its derived in-memory views. - -## Precedence and lifetime - -- The loose layer precedes every installation source, including installation mods. -- Within a layer, normal source precedence applies. Identical init records agree; - conflicting records in the same tier remain unresolved with diagnostics. -- Ped-init selection uses the record's model identity, not its containing filename. -- Lookups return overlay-local IDs; reading always routes to the original source. - Never persist these IDs or insert them into an installation index. -- Closing the context releases its loose resources, not the borrowed installation. -- Neither metadata resolution nor overlay mounting loads unrelated YMAP/YBN assets. - -## Changed exports and cancellation - -```python -from fivefury import CutsceneResolutionCancellation - -cancellation = CutsceneResolutionCancellation() -assets.remount( - "export/loose", - registrations=registrations, - cancellation=cancellation, -) -bundle = assets.resolve_cutscene("scene.cut", cancellation=cancellation) -``` - -Remount after changing export files, passing the current registrations again. -The replacement becomes active only after indexing and registration succeed. -A missing registration, unreadable archive or cancellation leaves the prior mount -active. Cancellation is checked before and after indexing and between registrations; -an individual archive-indexing call is not interrupted midway. - -Remounting discards derived expression/outfit state. Resolve a new bundle/catalog; -do not reuse assets from the old loose mount. They are rejected by the new mount. -Concurrent remount and resolution on the same overlay are not supported. - -Malformed metadata produces decode/resolution diagnostics rather than a fake init -record. Successful CPU resolution or expression evaluation does not certify -in-game playback, rendering or DLC mounting. diff --git a/docs/ASSET_SOURCES.md b/docs/ASSET_SOURCES.md deleted file mode 100644 index a9c3905..0000000 --- a/docs/ASSET_SOURCES.md +++ /dev/null @@ -1,43 +0,0 @@ -# Asset source provenance - -`AssetRecord.source_tier` identifies the semantic source using `AssetSourceTier`: - -| Tier | Source | -| --- | --- | -| `BASE` | Base installation content. | -| `UPDATE` | Update content outside DLC packs. | -| `DLC` | Content under a DLC pack path. | -| `MODS` | Installation content under `mods/`. | -| `OVERLAY` | Assets owned by a scoped `GameFileOverlay`. | - -Fallback records retain their installation tier. Overlay provenance applies to -both loose files and entries inside the overlay's RPFs. A standalone loose cache -uses its installation-relative paths; being a loose file alone does not make an -asset an overlay. - -Provenance is not a sorting key. `source_priority` is a separate ordering value: -an overlay's fallback BASE record can have priority 7 and still have tier `BASE`. -Never construct an enum from a sorting priority or use enum values to compare -overlay and installation precedence. - -Vehicle appearance sources use this same enum, replacing the former -`VehicleAppearanceSourceTier`. Import `AssetSourceTier` from `fivefury` or -`fivefury.cache`. - -```python -from fivefury import AssetSourceTier, GameFileOverlay - -with GameFileOverlay(export_directory, fallback=installation) as overlay: - appearance = overlay.resolve_vehicle_appearance("example_car") - for source in appearance.sources: - print(source.path, source.tier.name) - if source.tier is AssetSourceTier.OVERLAY: - print("Definition supplied by the export") -``` - -Variations and color definitions resolve independently. An installation-only -variation can therefore use colors overridden by the overlay, and both sources -remain visible in the result. Closing the overlay does not close its borrowed -installation cache. Remount after modifying export files. -Rescanning the borrowed installation invalidates the overlay's appearance index -before the next appearance query, including when the asset count is unchanged. diff --git a/docs/AUTHORING_OPERATIONS.md b/docs/AUTHORING_OPERATIONS.md deleted file mode 100644 index 6ca4bbb..0000000 --- a/docs/AUTHORING_OPERATIONS.md +++ /dev/null @@ -1,100 +0,0 @@ -# Authoring Operations - -CUT projects, asset exports and YCD builders accept an optional -`AuthoringOperation` for progress and cooperative cancellation. One token can -cover construction, validation and saving without changing asset encoding. - -```python -from fivefury import ( - AuthoringOperation, - AuthoringProgress, - YcdCutsceneBuilder, -) - -def receive_progress(progress: AuthoringProgress) -> None: - print(progress.stage, progress.asset, progress.completed, progress.total) - -operation = AuthoringOperation(progress=receive_progress) -builder = YcdCutsceneBuilder.create("scene", duration=5.0) -builder.prop("actor") -assets = builder.build_ycds(operation=operation) -``` - -`build_section`, `build_ycds`, `validate` and `save` accept `operation=`. -The operation is optional and does not change precision policy or generated bytes. - -## Threading And Cancellation - -- Callbacks execute synchronously on the authoring thread. Dispatch to your UI's - event queue rather than updating UI objects directly from the callback. -- Another thread can call `operation.cancel()`. The next checkpoint raises - `AuthoringCancelled`; a cancelled operation cannot be reused. -- Validation checks cancellation between sequence blocks. Encoding and resource - serialization complete their current unit of work before reaching a checkpoint. -- Saving validates every section before writing final files. Each replacement is - atomic. `CutsceneAssets.save` stages the complete batch before replacing files: - cancellation during preparation leaves existing outputs intact. Replacement is - non-cancellable; ordinary replacement failures roll back completed replacements. - This is not a filesystem-wide power-loss transaction. Direct YCD builder saves - still replace sections individually. -- Callbacks and cancellation never bypass validation or change tolerances. - -## Progress Units - -`AuthoringProgress` contains a typed `AuthoringStage`, an asset name, a completed -count and a total count. Counts belong to that stage and asset, not a global -percentage: building reports sections, validation reports constrained tracks in -one YCD, and writing reports output files. - -## Precision And Mutation - -Validation reads the serialized resource back before comparing samples. Native -comparison interpolates stored channels before reconstructing or normalizing -cached quaternions. Integer errors and fractional angular errors retain separate -diagnostics; fractional checks include sequence overlaps. - -Build reuse is scoped to one call. Editing tracks and building again constructs -and validates new assets. No precision result or serialized output is cached on -the mutable builder for later calls. - -## CUT Projects And Facial Tracks - -```python -from fivefury import ( - AuthoringOperation, CutsceneProject, Quaternion, Vector3, YcdFacialTrackSet, -) - -project = CutsceneProject.create("dialogue", duration=2.0, camera_cuts=[1.0]) -actor = project.scene.ped("speaker", model_name="cs_actor") -project.camera(position=Vector3(0, -3, 1), rotation=Quaternion()) -project.animate( - actor, - clip="speaker", - mover_position=Vector3(), - facial=YcdFacialTrackSet(controls={7: {0.0: 0.0, 1.0: 0.8, 2.0: 0.0}}), -) - -operation = AuthoringOperation(progress=receive_progress) -assets = project.build(operation=operation) -assets.validate(operation=operation).raise_for_errors() -paths = assets.save("output", operation=operation) -``` - -`facial=` accepts `YcdFacialTrackSet` on ped bindings only. Omit the body arguments -for a facial-only clip. The project derives the merged facial state and writes -`speaker_dual-0`, `speaker_dual-1`, etc.; the CUT streaming base remains `speaker`. -Omitting `facial=` preserves body-only authoring. Control IDs and values must -match the ped's expression assets; the sample ID above is not universal. - -Pass `context=BuildContext(...)` to `assets.validate`, `build_files` or `save` -to check model, skeleton and expression dependencies from loose assets or a game -cache. These validations are not proof of playback in the game. - -`CutsceneProject.build`, `CutsceneAssets.build`, `validate`, `build_files` and -`save` accept `operation=`. The same token reaches YCD construction, precision -validation and serialization. Checkpoints occur between sections, clips, tracks, -validation phases and output preparations. Compression, an individual asset -read and other current units finish before cancellation is observed. - -Progress counts are local to each unit of work and can restart on a later pass; -they are not a monotonically increasing percentage for the whole project. diff --git a/docs/AWC_PLAYBACK_CONTRACTS.md b/docs/AWC_PLAYBACK_CONTRACTS.md deleted file mode 100644 index 8f47202..0000000 --- a/docs/AWC_PLAYBACK_CONTRACTS.md +++ /dev/null @@ -1,51 +0,0 @@ -# AWC playback contracts - -`read_awc` decodes the container; reading successfully does not certify audio -playback. `Awc.validate()` checks stream metadata, encoded payload capacity and -streaming timelines. `to_bytes()` and `save()` enforce those checks before writing. -`save()` replaces the destination atomically after validation and serialization. - -```python -from fivefury import Awc - -bank = Awc.from_channel_mp3("dialogue", [left_pcm16, right_pcm16], sample_rate=48000) -bank.validate().raise_for_errors() -bank.save("dialogue.awc") -``` - -## Streaming - -- Every stream table is sorted by its 29-bit hash. Channel order is independent - and remains in STREAM_FORMAT. -- MP3 channel payloads begin on 16-byte boundaries. Encoded size excludes that - padding, and the shared packet-table region is aligned to 2048 bytes. -- PCM uses extended block headers and absolute packet sample positions. Only - intermediate blocks are padded to the nominal streaming block size. -- Legacy PCM/ADPCM fixed-block headers remain readable, including final packet - padding bounded by the codec's packet sample count. New authoring uses the - extended marker; validation does not mistake original legacy markers for MP3. -- Skipped samples represent the repeated prefix of an overlapping packet. They - are removed from decoded output and counted when checking block continuity. -- Sample capacity comes from payloads, not matching declared durations alone. -- Retail MP3 trimming can leave a larger pre-trim frame counter. Validation - permits a bounded final-packet trim and uses the actual decoded frame capacity. - -Playback validation covers fixed-width PCM/float data, ADPCM block dimensions and -step indices, mono MP3 frame seek tables, and MP3/PCM/ADPCM streaming timelines. -Other codec layouts are reported with `awc.codec.unsupported`; recognizing a -codec identifier does not establish its playback contract. - -## Preservation - -An imported, unchanged resource whose only validation errors are unsupported -codecs can retain its exact original bytes. This is preservation, not successful -playback validation. Editing its serialized fields invalidates that path. -Known malformed data is not exempt merely because it came from a file. - -Metadata-only containers can remain readable/writable with a no-playable-stream -warning. An incomplete audio stream with FORMAT but no DATA (or the reverse) is -an error, as are duplicate chunk types in an audio stream. - -These checks do not prove audio quality, correct REL references, native mounting -or successful in-game playback. They do not identify a crash without matching -runtime evidence. diff --git a/docs/STYLE_GUIDE.md b/docs/STYLE_GUIDE.md deleted file mode 100644 index 91a3f76..0000000 --- a/docs/STYLE_GUIDE.md +++ /dev/null @@ -1,292 +0,0 @@ -# FiveFury Style Guide - -This document is normative. New code and refactors must follow these rules even when an older module does not. - -## Design priorities - -1. Model the game data faithfully. -2. Make valid authoring straightforward and invalid state visible. -3. Prefer one obvious operation over aliases and compatibility wrappers. -4. Keep binary policy in writers and domain policy in typed models. -5. Minimize code, indirection, allocations, and repeated work. - -## Public API grammar - -FiveFury models are ordinary typed Python objects. Their public collections are ordinary typed collections. - -```python -entity = EntityDef( - archetype_name="harbor_lamp", - position=Vector3(10.0, 20.0, 5.0), -) -ymap.entities.append(entity) -``` - -An aggregate may expose a singular noun as a convenience factory. The factory constructs, registers, and returns the object. - -```python -entity = ymap.entity("harbor_lamp", position=Vector3(10.0, 20.0, 5.0)) -bone = skeleton.bone("root") -physics = ymap.physics_dictionary("harbor_collision") -``` - -Do not expose `add_*`, generic `add()`, or duplicate `create_*` aliases. Existing objects use `append`, `extend`, mapping assignment, or a domain relationship verb. A noun factory must not silently perform unrelated derivations. - -One-to-one relationships use properties when assignment is sufficient: - -```python -ydr.bound = collision -model.skeleton = skeleton -``` - -Use a verb only when the operation has domain behavior that property assignment cannot express. - -| Verb | Required meaning | -| --- | --- | -| `ensure_*` | Return an existing component or create the minimum empty component. | -| `derive_*` | Produce data from authoritative source data. | -| `recalculate_*` | Replace existing derived values. | -| `normalize_*` | Canonicalize representation without inventing content. | -| `bind_*` | Establish and validate a semantic relationship. | -| `resolve_*` | Find a reference without modifying source data. | -| `validate` | Inspect only; never repair or mutate. | -| `build` | Finalize normalized in-memory state for serialization. | -| `save` | Build, validate, serialize, and atomically replace the destination. | - -Do not use `get_*` for properties, `set_*` for ordinary assignment, or `do_*` for any public operation. - -## Cross-asset authoring - -Use `AssetRef[T]` for typed cross-file references, `AssetSet` for a named collection, and `BuildContext` for target and resolution state. Do not pass expanding groups of unrelated `ytyps=`, `ybns=`, `cache=`, `game=`, or shader lookup arguments through every layer. - -```python -assets = AssetSet() -assets["stream/harbor_lamp.ydr"] = drawable - -context = BuildContext(game=GameTarget.GTA5, assets=assets, strict=True) -drawable_ref = AssetRef("harbor_lamp", Ydr) -resolved = context.resolve(drawable_ref) -``` - -References preserve names and type intent. Binary writers convert them into the hashes, indices, pointers, or names required by the format. - -## Diagnostics - -Public validation uses `ValidationReport` and typed `Diagnostic` values. A diagnostic has a stable code, severity, message, asset, and field path. Do not return bare strings, mix warnings with errors, print from validators, or mutate objects while validating. - -```python -report = context.validate(asset) -report.raise_for_errors() -``` - -Errors describe violated invariants. They must not promise runtime behavior that has not been tested in the game. - -## Binary boundaries - -- Readers decode bytes into typed models and preserve genuinely unknown data when lossless preservation is possible. -- Writers serialize typed models, calculate addresses and fixups, enforce packed limits, and write atomically. -- Builders derive nontrivial structures such as BVHs, bounds, manifests, ownership, and runtime hashes. -- High-level helpers coordinate existing public models; they do not duplicate parsers or writers. -- Edition-specific behavior is selected through `GameTarget`, never inferred from a local machine path. - -Never expose raw dictionaries when a stable structure is known. Never guess an unknown field merely to make a fixture pass. - -## Duplication and module boundaries - -Before adding code, search the full source tree for the operation and its underlying math, hashing, XML, resource, texture, or graph logic. - -- Math belongs in the shared math/vector module. -- Values with named mathematical components use the shared nominal types - (`Vector2`, `Vector3`, `Vector4`, `Quaternion`, `Aabb2`, and `Aabb3`), never - anonymous float tuples or local aliases. Public models, readers, builders, - and writers must preserve those types end to end. -- Mathematical code uses named fields and domain operations such as `value.x`, - `value.normalized()`, `left.cross(right)`, and `rotation.rotate(position)`. - Positional component access is reserved for binary, NumPy, and foreign-function - boundaries where the external representation is inherently indexed. The nominal - value objects themselves are iterable for those boundaries but are not indexable - sequences. -- Hashing belongs in the hashing module and uses the native implementation where available. -- XML primitives belong in the shared XML module. -- Resource layout, pointer, page, and fixup logic belongs in the resource layer. -- Shared drawable behavior belongs in `drawable`, not separately in YDR, YDD, YFT, or CDR. -- Format modules contain only format-specific models and policy. - -Do not create a utility for a single call site. Extract behavior when it expresses a stable concept or removes real duplication. - -## Simplicity rules - -- Prefer data flow over flags that select unrelated behavior. -- Prefer a short typed object over a string-keyed dictionary. -- Prefer iteration over repeated near-identical branches. -- Prefer early validation over defensive fallback chains. -- Delete unreachable branches and superseded APIs instead of deprecating them. -- Do not catch `Exception` unless the boundary explicitly converts arbitrary plugin or file failures into diagnostics. -- Do not retain Python fallbacks for native code solely for obsolete tests. -- Avoid methods that only forward arguments under another name. -- Avoid comments that restate code. Comments explain binary evidence, invariants, or non-obvious constraints. - -Line count is not a target by itself, but repeated ceremony is a design defect. If a helper needs many aliases, coercion branches, optional names for the same value, or boolean mode switches, redesign the model. - -## Performance rules - -- Do not repeat parsing, hashing, decompression, graph traversal, or bounds calculation inside loops. -- Build indexes once for repeated lookup. -- Stream large files and archives where the format permits it. -- Keep large binary buffers as `bytes`, `memoryview`, or native containers rather than lists of Python integers. -- Move measured CPU-heavy loops to the native extension when the ABI can stay small and generic. -- Benchmark before and after optimization; do not claim improvement from code shape alone. - -## Native ABI verification - -An `abi3` wheel must be tested on the minimum supported Python, even when built -with newer headers. Run `tests/runtime/test_native_abi_buffers.py` against the built wheel -with `FIVEFURY_ABI_TEST_PYTHON` pointing to Python 3.11. Build-host tests alone do -not verify the compatibility promised by the wheel tag. - -Ordinary PRs build one wheel and verify it on Python 3.11. Release candidates -use the complete supported Python matrix before publication; use a `release/` -branch or dispatch the test workflow with `full_matrix`. Building a wheel in -CI is validation, not permission to publish it. - -## Test contracts - -- Unit tests must run without a game installation or private asset corpus. Generate - synthetic inputs for general contracts; retain independent binary assertions - and real-file tests where format fidelity needs external evidence. -- Import the current API directly. Do not search alternative symbol names, catch - missing API errors, or condition assertions on a method being present. -- Mark external tests with `integration`, cross-interpreter checks with `abi`, - and timing measurements with `performance`. Selection is explicit through - `--suite`; a selected test must fail if its prerequisite is missing. -- Do not use `skip`, `skipif`, `importorskip`, or empty parameter sets to hide a - missing requirement. Platform-specific tests use `requires_platform`. -- Functional tests assert observable results and call counts, not relative wall - time or artificial sleeps. Performance measurements belong in benchmarks. -- Source and wheel runs exercise the same test files. Subprocess tests must use - the same package installation as their parent, not an incidental editable copy. -- Track all test sources and synthetic fixtures. Never commit private game assets - or local paths; configure external data through the documented environment. - -## Compatibility policy - -FiveFury favors a coherent current API over compatibility with accidental historical APIs. Breaking refactors must: - -1. Remove the old symbol and its internal uses in the same change. -2. Update tests, examples, type exports, and documentation. -3. Avoid aliases, warnings, forwarding wrappers, and hidden fallback behavior. -4. Record the breaking change concisely in the changelog. - -Tests validate the intended API. They are not a reason to preserve obsolete code. - -## Migration examples - -The removed forms below are shown only to make the API change explicit. They are not compatibility aliases. - -### Typed collections and noun factories - -Before, a YMAP mixed generic dispatch, format-specific insertion methods, and duplicate factories: - -```python -ymap.add(entity) -ymap.add_entity(entity) -created = ymap.create_entity("harbor_lamp", position=(10.0, 20.0, 5.0)) -``` - -Now existing values use the collection and convenience construction uses one singular noun: - -```python -ymap.entities.append(entity) -created = ymap.entity( - "harbor_lamp", - position=Vector3(10.0, 20.0, 5.0), -) -``` - -### Relationships and ordinary assignment - -Before, relationship setters and insertion verbs obscured ordinary object ownership: - -```python -ydr.set_bound(collision) -skeleton.add_bone(Bone("root")) -ydr.add_light(light) -``` - -Now assignment expresses ownership, while singular nouns construct or register children: - -```python -ydr.bound = collision -root = skeleton.bone("root") -ydr.light(light) -``` - -### Cross-asset operations - -Before, every layer forwarded an expanding parameter group: - -```python -ymap.to_bytes(ytyps=[ytyp], ybns={"interior": ybn}) -manifest = build_ymf_for_ymaps(ymaps, cache=cache, ytyps=[ytyp], ybns=ybns) -``` - -Now assets and resolution state travel together: - -```python -assets = AssetSet() -assets["stream/interior.ytyp"] = ytyp -assets["stream/interior.ybn"] = ybn -context = BuildContext(assets=assets, cache=cache) - -data = ymap.to_bytes(context=context) -manifest = build_ymf_for_ymaps(ymaps, context=context) -``` - -### CUT authoring - -Before, construction mixed `add_*`, `create_*`, and generic dispatch: - -```python -scene.add_prop("stage", model="stage01") -scene.add_camera("main_camera") -scene.add_event(camera_cut) -``` - -Now domain nouns and an explicit timeline operation describe the same scene: - -```python -stage = scene.prop("stage", model="stage01") -camera = scene.camera("main_camera") -scene.timeline_event(camera_cut) -``` - -### Archives and validation - -Before, archives had multiple insertion aliases and validators exposed unrelated result shapes: - -```python -archive.add_file("stream/model.ydr", data) -archive.add_asset("stream/model.ydr", data) -issues = asset.validate() -``` - -Now archive insertion has one name and validation has one structured result: - -```python -archive.file("stream/model.ydr", data) -report = context.validate(asset) -report.raise_for_errors() -``` - -## Review checklist - -- Is there exactly one public way to perform the operation? -- Does every method name describe domain intent rather than list mechanics? -- Are all inputs typed and are stable game values enums? -- Is validation non-mutating and diagnostic-driven? -- Is serialization atomic and performed only after validation? -- Has shared logic been reused rather than copied? -- Are unknown values preserved or rejected instead of guessed? -- Are hot loops indexed, batched, streamed, or native where justified? -- Were obsolete APIs, tests, exports, and comments deleted? -- Does the implementation remain faithful to both Legacy and Enhanced where the format differs? diff --git a/docs/YCD_CHANNEL_ENCODING.md b/docs/YCD_CHANNEL_ENCODING.md deleted file mode 100644 index d172d45..0000000 --- a/docs/YCD_CHANNEL_ENCODING.md +++ /dev/null @@ -1,137 +0,0 @@ -# YCD channel encoding - -`YcdChannelEncodingPolicy` selects codecs and optional read-back accuracy limits. -The builder policy is the default; an individual `track(..., channel_policy=...)` -replaces it for that track only. - -| Track | RETAIL, varying components | RAW_FLOAT, varying components | -| --- | --- | --- | -| Bone/mover translation and rotation | 16-bit quantized floats | 32-bit floats | -| Facial translation, rotation and scale | 16-bit quantized floats | 32-bit floats | -| Other tracks, including camera channels | 32-bit floats | 32-bit floats | - -Static tracks and constant components keep their static codecs. Quaternion -packing is a separate choice: the channel policy does not disable cached -reconstruction or change its runtime interpolation order. Compression does not -rename tracks or clips, remove controls, alter timing, or change section and -sequence boundaries. - -## Mixed precision - -```python -from fivefury import ( - Vector3, - YcdAnimationTrack, - YcdChannelEncoding, - YcdChannelEncodingPolicy, - YcdCutsceneBuilder, -) - -high = YcdChannelEncodingPolicy( - encoding=YcdChannelEncoding.RAW_FLOAT, - maximum_error=2e-5, - maximum_angular_error_degrees=0.05, -) -face = YcdChannelEncodingPolicy( - encoding=YcdChannelEncoding.RETAIL, - maximum_error=1e-5, - maximum_angular_error_degrees=0.05, -) -builder = YcdCutsceneBuilder.create("scene", duration=1.0, channel_policy=high) -builder.track( - "actor_dual", - track=YcdAnimationTrack.MOVER_TRANSLATION, - samples={0.0: Vector3(), 1.0: Vector3(100, 0, 0)}, -) -builder.track( - "actor_dual", - track=YcdAnimationTrack.FACIAL_TRANSLATION, - bone_id=7, - samples={0.0: Vector3(), 1.0: Vector3(0, 0.02, 0)}, - channel_policy=face, -) -ycds = builder.build_ycds() -``` - -Use the same per-track policy for `FACIAL_ROTATION` and `FACIAL_SCALE`. -Facial rotations take `Quaternion` samples; translation and scale take `Vector3`. -Keep the original facial control IDs and merged body/face clip names. - -## Quaternion representation and precision - -Scalar encoding and quaternion representation are independent. RAW_FLOAT does -not disable cached reconstruction, and constant rotations normally use -STATIC_QUATERNION: three float32 components plus reconstructed positive W. -When W is close to zero, rounded XYZ can cause a component error larger than -1e-5 even without varying channels. Cached varying rotations can also deviate -from source quaternion interpolation between frames before any quantization. - -`track(..., quaternion_encoding=YcdQuaternionEncoding.EXPLICIT)` stores all four -components for that rotation track, including four STATIC_FLOAT channels when -constant. It overrides the builder setting without changing other tracks. There -is no automatic representation change based on a failed precision check. - -For strict component and angular bounds, a consumer may explicitly choose: - -```python -from fivefury import YcdQuaternionEncoding - -strict = YcdChannelEncodingPolicy( - encoding=YcdChannelEncoding.RAW_FLOAT, - maximum_error=1e-5, - maximum_angular_error_degrees=0.05, -) -builder.track( - "actor_dual", - track=YcdAnimationTrack.FACIAL_ROTATION, - bone_id=control_id, - samples=original_quaternions, - channel_policy=strict, - quaternion_encoding=YcdQuaternionEncoding.EXPLICIT, -) -``` - -The cost is an extra stored quaternion component, not changed source samples. -Use RAW_FLOAT explicitly for translations whose range cannot meet a requested -bound with 16-bit quantization. EXPLICIT controls quaternion layout only; it does -not raise the scalar bit depth when the channel policy is RETAIL. - -Alternatively, applications whose rotation contract is angular rather than -component-based may set `maximum_angular_error_degrees` and leave the rotation's -`maximum_error=None`. That is a different, explicitly chosen contract, not a fix -for a failed component bound. Vector and quaternion components have different -units; the application owns that decision. FiveFury does not drop either limit -when both are requested or substitute a different interpolation during validation. - -## Accuracy contract - -- `maximum_error` limits the largest absolute component difference, not vector - length or world-space displacement. Quaternion comparisons account for the - equivalence of opposite signs. -- `maximum_angular_error_degrees` independently limits rotation error. -- Both limits cover integer frames and subframes at 0.25, 0.5 and 0.75 of every - interval, including the stored overlap at a physical sequence boundary. -- Validation uses encoded and decoded values, including float32 quantization - parameters. It does not certify just the original in-memory samples. -- Cached quaternion validation interpolates stored components before - reconstruction/normalization, matching that codec's behavior. - -These are checks at specified sample locations, not a proof of an error bound at -every real-valued time. A component-only quaternion policy still checks subframes; -an angular bound is not required to enable those checks. - -`validate()` reports precision errors without weakening the requested limits. -Integer component failures use `ycd.channel_precision.error_exceeded`; subframe -component failures use `ycd.channel_precision.subframe_error_exceeded`. Each -diagnostic identifies its worst frame within the generated section. Quaternion -component failures also report the maximum angular error for the same sampling -phase, even when that angular error meets its independent limit. The two maxima -need not occur at the same frame. - -`build_ycds()` and `save()` reject violations before returning or publishing the -outputs. There is no automatic fallback to RAW_FLOAT or lower precision: choose -another policy explicitly if the requested codec cannot meet the limit. -RAW_FLOAT also has float32 rounding and is not an unlimited-precision mode. - -Existing YCD bytes are not retroactively compressed. Rebuild the CUT/YCD assets -from the original authored samples after adopting a different channel policy. diff --git a/docs/YCD_PLAYBACK.md b/docs/YCD_PLAYBACK.md deleted file mode 100644 index 726c2a4..0000000 --- a/docs/YCD_PLAYBACK.md +++ /dev/null @@ -1,77 +0,0 @@ -# Compiled YCD playback - -Mutable YCD models remain the authoring interface. Compile once when beginning -repeated playback, not once per frame: - -```python -from fivefury import read_ycd - -ycd = read_ycd("scene-0.ycd") -clip = ycd.get_clip("actor_dual-0") -sampler = clip.compile() -tracks = sampler.evaluate_tracks_at_time(1.234) -``` - -`YcdAnimation.compile()` returns `YcdAnimationSampler`, with the existing -`evaluate_tracks(frame, track=None, interpolate=True)` shape. A clip returns -`YcdClipSampler`, with `evaluate_tracks_at_time` and `evaluate_tracks_at_phase`. -Results remain dictionaries keyed by `(bone_id, track_id)`, containing nominal -`Vector4` or `Quaternion` objects. There are no application-facing native handles. - -## Lifetime and edits - -- Compilation copies decoded channel cycles into owned native float64 buffers. - It does not retain the editable model, reader, RPF or external buffer. -- A sampler is immutable. Source edits deliberately do not change it; recompile - after editing channels, bindings, sequence structure or clip timing. -- Dropping the last sampler reference releases its storage. There is no global - plan cache, sampled-frame cache or retained per-seek state. -- `sample_bytes` reports copied channel data, excluding metadata and output - objects. Static channels keep one value per component rather than a full frame - array. Indirect channels retain their decoded frame cycle. -- Samplers can be shared by concurrent actors. Evaluation releases the GIL for - native sampling and allocates independent output state per call. -- Multiple `YcdClipSampler` instances can explicitly share one compiled - `animation`, with independent duration/loop metadata. - -Do not modify a model concurrently with its compilation. An already compiled -sampler is independent of subsequent mutations and seeks. - -## Sampling contract - -The compiled path uses decoded channel values; it does not requantize or change -the animation. Opcode 7/8 reconstruction or normalization occurs after scalar -interpolation, using the current block's overlapping sample. Ordinary quaternion -tracks retain their existing shortest-path normalized interpolation. - -Integer arguments and `interpolate=False` retain integer evaluation behavior, -including cyclic channel lookup. Fractional requests retain the existing endpoint -clamping and missing-track rules. Nonfinite or unrepresentably large native frame -indices are rejected. - -Clip time and phase conversion are shared with `YcdClipAnimation`. The existing -methods clamp to clip duration/phase; they do not apply an implicit rate change -or wrap solely because the loop flag is set. A caller that currently computes -looped phase or time must keep doing so. This is a performance path, not a change -to the runtime interpretation of clip timing metadata. - -The compilation boundary expects valid decoded or built channel layouts. A -cached quaternion opcode without a matching cached sequence, an invalid omitted -component or an incomplete cached component set is rejected rather than guessed. - -## Verification - -Unit tests compare mutable and compiled results at integer/fractional frames, -sequence overlaps, out-of-range seeks, channel cycles and missing keys. They also -cover edits, detached lifetime, concurrent evaluation and output construction. -Performance cases are in `tests/performance/bench_ycd_playback.py`. - -An external comparison of YCD names associated with indexed vanilla CUTs covered -10,275 Enhanced resources (195,904 animation clips) and 9,127 Legacy resources -(179,030 animation clips), including archive variants. The tested integer, -fractional, block-boundary and clip-time positions produced no numerical -differences from the mutable sampler. This is sampled differential coverage, not -a claim that every possible time or custom resource has been proven equivalent. - -Library sampling improvements are not viewer FPS predictions: rendering, -skinning, expression evaluation and scheduling remain separate workloads. diff --git a/docs/YED_PLAYBACK.md b/docs/YED_PLAYBACK.md deleted file mode 100644 index 1e834a9..0000000 --- a/docs/YED_PLAYBACK.md +++ /dev/null @@ -1,45 +0,0 @@ -# Compiled YED evaluation - -For repeated playback, compile expression selection and skeleton defaults once: - -```python -from fivefury import read_yed - -yed = read_yed("expressions.yed") -evaluator = yed.compile(expression_names, skeleton=skeleton) -result = evaluator.evaluate(tracks, time=seconds, delta_time=delta_seconds) -``` - -`YedEvaluator` returns the same `YedEvaluationResult` as `evaluate_yed`: typed -tracks, output tracks, variables, expression names, issues and native-attachment -diagnostics. It is not a private native handle. The existing mutable convenience -function also uses the native typed boundary, but checks complete program/default -content before reusing its bounded program cache. - -## Ownership - -- A compiled evaluator owns a native snapshot of programs and skeleton defaults; - no source YED/skeleton is retained. Recompile after edits or expression-set changes. -- Compilation captures unresolved-expression and zero-signature diagnostics. - Those diagnostics describe the snapshot, not subsequent source edits. -- Evaluations have independent native frames and release the GIL during the VM - batch. There is no implicit per-actor state or cached last frame. -- Pass a separate mutable `variables` mapping per actor only when persistent - variable state is intended. Without it, evaluation starts with fresh variables. -- Input Vector3, Vector4, Quaternion and scalar/iterable values cross the boundary - directly. The native result contains final nominal Vector4 values, avoiding - repeated Python dictionary/tuple conversions. -- Output values may share immutable Vector4 objects with matching final tracks - within the same result. Result mappings and evaluation state are independent. - -## Editable models - -`evaluate_yed` detects changes to operands, instruction contents, stream metadata -and skeleton defaults, even when object/list identities and lengths stay unchanged. -This requires inspecting source content and is not the high-throughput playback -path. Use `compile()` once for repeated immutable playback instead of depending on -an incomplete cache signature or invoking compilation for every frame. - -CPU evaluation and native attachment remain separate checks. A numerical output -does not prove that the expression can attach to a runtime actor. Compilation does -not erase diagnostics, repair malformed assets or certify in-game behavior. diff --git a/docs/YED_RUNTIME_CONTRACTS.md b/docs/YED_RUNTIME_CONTRACTS.md deleted file mode 100644 index c0baf3b..0000000 --- a/docs/YED_RUNTIME_CONTRACTS.md +++ /dev/null @@ -1,211 +0,0 @@ -# YED native authoring contracts - -CPU expression evaluation and native expression attachment are different checks. -An executable expression with a zero signature can produce numerical results in -the CPU evaluator but is not a valid native authoring result. - -## Finalize explicitly - -```python -from fivefury import ( - MetaHash, - YedExpression, - YedInstruction, - YedInstructionType as Op, - YedStream, - YedTrackFormat, - create_yed, -) - -expression = YedExpression.create("facial_transform") -expression.streams = [ - YedStream( - name_hash=MetaHash("main"), - depth=0, - data1=b"", - data2=b"", - data3=b"", - instructions=[ - YedInstruction(Op.TRACK_VALID, operands={ - "bone_id": 123, "track": 25, "format": YedTrackFormat.VECTOR3, - }), - YedInstruction(Op.JUMP_IF_FALSE, operands={"instruction_offset": 2}), - YedInstruction(Op.TRACK_GET, operands={ - "bone_id": 123, "track": 25, "format": YedTrackFormat.VECTOR3, - }), - YedInstruction(Op.TRACK_SET, operands={ - "bone_id": 123, "track": 0, "format": YedTrackFormat.VECTOR3, - }), - YedInstruction(Op.END), - ], - ), -] - -yed = create_yed(expression) -yed.recalculate_runtime_contract() -yed.validate_runtime_contract().raise_for_errors() -yed.save("facial_transform.yed") -``` - -The same operation is available on `YedExpression` when finalizing one expression. -Finalization prepares detached state first: an error does not leave a partially -rewritten expression or dictionary. Run it again after editing instructions. -It derives: - -- Separate input/output tracks and their accelerated operand indices. -- The incremental CRC32 signature, including repeated accesses. -- Variable indices and motion descriptors for supported instructions. -- Stack depth, maximum stream size and rebuilt data/opcode buffers. -- Branch byte offsets from each relative instruction target. - -Do not assign a donor signature, hash the expression name, or force a nonzero -placeholder. Identical ordered DOF interfaces can legitimately share a signature -even when expression names or calculations differ. - -## Direction and traversal - -`YedTrack.is_input` is bit 7; the lower seven bits describe the track format. -It does not request bone remapping. Track factories accept `is_input=True`; -the old `remap` argument and `remap_flag` property are removed. - -Track deduplication uses **direction, channel and bone ID**. The CRC includes -every visited access as a little-endian uint32 `(channel << 16) | bone_id`, -with a zero initial CRC. It does not include only unique tracks. - -Traversal is parent-before-children, not raw bytecode order. For the guarded -example above, the signature visits VALID, SET, GET. Nested supported stack -expressions and structured forward branches are reconstructed before traversal. -Packed linear sources are returned to their original four-lane source order. - -## Validation and preservation - -`validate_runtime_contract()` does not mutate anything. It checks native signature, -direction, track/variable indices, branch offsets and stream-buffer consistency. -`validate()` includes these checks together with structural validation. Writers -reject incomplete authored contracts before replacing the destination file. - -An unmodified imported resource can retain its original bytes when some operations -or its pre-packing signature cannot be reproduced. This is a **preservation path**, -not certification of native playback. A zero-signature executable expression is -still rejected. Unverifiable original signatures are reported and are not silently -replaced by recalculation. - -Unknown opcodes, overlapping/unstructured branches, duplicated computed values -and ambiguous statement/value branch mixtures are not guessed. Preserve those -original programs or provide supported authored instructions. Direct edits to a -read YED invalidate its original-byte fast path; the writer cannot silently discard -them anymore. - -## Resource placement - -Semantic validity is separate from memory placement. `yed.validate()` also checks -the serialized resource layout of an unchanged import: owned blocks must fit in -individual RSC chunks, pointers and capacities must be valid, streams must be -aligned, and page metadata must reserve all system and graphics allocations. -`validate_runtime_contract()` checks expression semantics, not resource placement. - -New exports use the shared block-aware resource packer. Complete streams -(header and all parameter buffers), arrays, expressions and strings are moved as -indivisible allocations. Only declared pointer fields are relocated; hashes, -floating-point literals and other non-pointer data are not scanned for addresses. - -Reading an older unsafe resource remains possible for inspection. Saving it -unchanged is rejected before replacing the destination. Regenerate supported, -reproducible expression contracts with the existing API: - -```python -from fivefury import read_yed - -yed = read_yed("original.yed") -report = yed.validate() -yed.recalculate_runtime_contract() -yed.save("regenerated.yed") -``` - -This preserves track semantics while rebuilding resource placement. Unknown or -unverifiable imported programs are not guessed: their original bytes may only be -preserved when the resource layout is safe. A target-edition override does not -bypass validation. No consumer should patch runtime indices or remove aliases to -compensate for an unsafe serialized layout. - -## Frame indices and CPU diagnostics - -`YedFrameLayout` describes the complete runtime frame, not a skeleton palette. -Derive it from the union of actual frame channels (skeleton DOFs, animation and -expression extras), or inspect a complete captured layout using `YedFrameDof`: - -```python -from fivefury import YedFrameLayout, YedTrack - -frame = YedFrameLayout.derive([ - YedTrack.vector3(7, 0), - YedTrack.quaternion(7, 1), - YedTrack.vector3(7, 25), -]) -frame.validate().raise_for_errors() -indices = expression.resolve_frame_indices(frame) -``` - -`derive` merges input/output references to the same physical DOF, rejects format -conflicts, sorts channels by track and bone ID, and packs each format with its -runtime padding. The final two 16-byte slots are reserved for read/write sentinels. -`buffer_size` includes these slots and must fit uint16. Validation rejects -misaligned, overlapping, noncanonical and out-of-frame DOFs rather than repairing -them. An empty frame still occupies 32 bytes. - -A matching **typed** DOF supplies its frame offset; an absent or differently typed -DOF uses `read_only_offset` for input tracks and `write_only_offset` for outputs. -Those two sentinel roles must not be confused with bone-ID remapping. -They are offsets of real storage, not `0xFFFF`. A missing channel does not by -itself imply an invalid accelerator address. This API describes layout only; -the runtime must initialize frame values and sentinel contents. - -Full `TRACK_GET`/`TRACK_SET` operations access 16 bytes; declaring their format as -`FLOAT` does not change the native instruction's access width. Scalar channels -must use `TRACK_GET_COMP`/`TRACK_SET_COMP` with component zero. Relative full-vector -operations are likewise not scalar operations. Quaternion component operations -use Euler XYZ components 0 through 2, not a fourth quaternion component. -These invariants are checked during contract derivation, validation and export, -including unchanged imports; preservation does not authorize an unsafe access. - -`frame.signature` is the runtime checksum of the sorted DOFs. Its value is not -proof of identity: caches must retain the full layout and invalidate on changes. -Do not pass an incomplete dump as a complete layout or infer absent DOF records. -The previous `resolve_frame_indices(dofs, read_only_offset=..., write_only_offset=...)` -form is removed; pass a validated `YedFrameLayout` instead. - -Inspect a previously cached or captured accelerator with -`expression.validate_frame_binding(frame, indices)`. It checks the expression's -global track contract, logical table length, alignment, bounds and every expected -DOF/sentinel offset. Stream operands index this global table; do not restart or -concatenate index numbering for each stream. Alias expressions with identical -track contracts may use the same mapping; names alone are not cache identities. - -The runtime cache lookup combines the frame signature in the upper 32 bits and -the expression signature in the lower 32 bits. Both are checksums, not unique -identifiers. Retain the complete frame DOFs and directional expression tracks -when caching in a consumer. Invalidate when either changes, even if the signature -or table length happens to remain the same. This validator recomputes the -expected mapping; an aligned, in-range offset is still wrong if it selects a -different channel or the wrong sentinel. - -For captures, supply only the logical expression entries, not allocator padding. -`None` marks unavailable captured entries and produces an explicit diagnostic; -it does not substitute a sentinel. A valid report certifies only the supplied -layout and table, not the lifetime of a pointer, contents of uncaptured memory, -or the values initialized by the game. Offline YED validation cannot certify -an accelerator which is allocated later by the runtime. - -The synthetic binding regressions cover both editions, large multi-stream -expressions and alias names. To test an external Enhanced YED with at least -500 tracks and five streams, set `FIVEFURY_TEST_YED_FRAME_CORPUS` to a directory of such YEDs -and select `tests/animation/yed/test_yed_frame_integration.py --suite integration`. -That test checks the expression-derived layout, not a captured creature frame. - -`evaluate_yed(...).issues` describes numerical/VM problems. -`evaluate_yed(...).native_diagnostics` separately reports the zero-signature -attachment gate. That fast check is not full contract validation: use -`validate_runtime_contract()` before export, including after any edits. - -Passing these checks does not establish skeleton compatibility, animation routing, -frame availability or in-game playback. Runtime replay is still required. diff --git a/docs/YMAP_PLACEMENTS.md b/docs/YMAP_PLACEMENTS.md deleted file mode 100644 index ec64698..0000000 --- a/docs/YMAP_PLACEMENTS.md +++ /dev/null @@ -1,57 +0,0 @@ -# YMAP Placements - -## Entity Flags - -`YmapEntityFlags` preserves all raw 32-bit flags, including unnamed bits. Reading -and writing do not shift or translate the numeric value. - -The declared entity-definition flags occupy bits 0–6 and 15–29. For example, -`DRAWABLELODUSEALTFADE` is `0x8000`, and `ONLY_RENDER_IN_WATER_REFLECTIONS` is -`0x08000000`, not `0x80000`. - -Two different sets of light flags must not be confused: - -| Purpose | Static Shadows | Dynamic Shadows | Ignore Day/Night | -| --- | --- | --- | --- | -| Declared entity-definition names | `LIGHTS_CAST_STATIC_SHADOWS` (`0x80000`) | `LIGHTS_CAST_DYNAMIC_SHADOWS` (`0x100000`) | `LIGHTS_IGNORE_DAY_NIGHT_SETTINGS` (`0x200000`) | -| IPL bits read by the entity runtime | `IPL_LIGHTS_CAST_STATIC_SHADOWS` (`0x80`) | `IPL_LIGHTS_CAST_DYNAMIC_SHADOWS` (`0x100`) | `IPL_LIGHTS_IGNORE_DAY_NIGHT_SETTINGS` (`0x200`) | - -`1572864` (`0x180000`) names the two declared shadow flags. It is not a -reflection-only mask, but the labels alone do not mean the runtime reads those -bits for light-shadow behavior. The IPL names express that distinction explicitly. - -These layouts are consistent with the Legacy definitions and the inspected -Enhanced entity initialization code. They are not evidence that a particular -flag causes or fixes rendering flicker. - -Code using incorrectly positioned symbolic constants from older releases must -be reviewed when regenerating assets. Numeric values already stored in files -remain untouched; FiveFury does not guess their original intent. - -## Stored And World Transforms - -`EntityDef.rotation` is the serialized placement quaternion, not automatically -the world-space orientation. Use `world_rotation(archetype)`, -`world_scale(archetype)` and `world_bounds(local_bounds, archetype)` when deriving -world data; these methods do not mutate the stored values. - -- Ordinary entities conjugate the stored quaternion when using a full transform. -- A small X/Y tilt uses the runtime's heading-only path unless `FULLMATRIX` or - an animated archetype with a clip dictionary requires a full transform. -- The heading-only path uses W and the sign of Z, rather than an Euler yaw - extracted from a full quaternion. The full-transform threshold is the runtime - float32 value of 0.05, with a strict greater-than comparison. -- Fragment placements ignore definition scaling. Drawable placements use XY/Z - scales independently. -- MLO parents use the stored orientation directly and unit scale. MLO child - entities still use ordinary `EntityDef` decoding before their parent transform. - -Provide the archetype to account for animated and fragment-specific behavior. -Without one, ordinary methods describe a static, non-fragment placement. -`Ymap.recalculate_extents(context=...)` resolves loose archetypes from its context -and uses these methods. LOD-light generation uses the same decoded placement; -its optional `archetype=` argument supplies animation and fragment metadata. - -Extents cover the placed archetype AABB, not every possible future animated pose. -These rules do not establish that unrelated streaming or rendering artifacts are -caused by placement bounds. From bce4ae07e42e3564f3ac1c8a15f3c99601ac796b Mon Sep 17 00:00:00 2001 From: MrVicho13 Date: Wed, 9 Sep 2026 05:51:44 -0300 Subject: [PATCH 2/2] Optimize drawable IO with native vectors and per-save geometry preparation --- fivefury/_native.py | 5 + fivefury/vector.py | 44 +++++ fivefury/ydd/writer.py | 14 +- fivefury/ydr/builder.py | 15 +- fivefury/ydr/prepare/bounds.py | 23 +-- fivefury/ydr/prepare/encoding.py | 20 ++- fivefury/ydr/prepare/mesh.py | 28 ++-- fivefury/ydr/prepare/splitting.py | 13 +- fivefury/ydr/reader.py | 12 +- fivefury/ydr/write_buffers.py | 7 +- fivefury/yft/writer.py | 13 +- native/bounds/python_conversions.cpp | 14 ++ native/drawable/bindings.h | 2 + native/drawable/vector_bindings.cpp | 66 ++++++++ native/drawable/vertex_bindings.cpp | 18 +- native/drawable/vertex_decode.cpp | 158 +++++++++++------- native/python/float_rows.h | 32 ++++ native/python/module.cpp | 2 + native/python/vector_factory.h | 46 +++++ .../drawable/ydr/test_ydr_prepared_bounds.py | 33 +++- tests/runtime/test_point_buffers.py | 94 +++++++++++ tests/runtime/test_vector_materialize.py | 63 +++++++ 22 files changed, 606 insertions(+), 116 deletions(-) create mode 100644 native/python/float_rows.h create mode 100644 native/python/vector_factory.h create mode 100644 tests/runtime/test_point_buffers.py create mode 100644 tests/runtime/test_vector_materialize.py diff --git a/fivefury/_native.py b/fivefury/_native.py index b5fc53e..f702606 100644 --- a/fivefury/_native.py +++ b/fivefury/_native.py @@ -572,6 +572,8 @@ def _ydr_decode_vertex_buffer( types_value: int, component_offsets: tuple[int, ...] | None, ) -> dict[str, object]: + from .vector import Vector2, Vector3, Vector4 + return _ffi.ydr_decode_vertex_buffer( data, int(vertex_count), @@ -579,6 +581,9 @@ def _ydr_decode_vertex_buffer( int(flags), int(types_value), component_offsets, + Vector2, + Vector3, + Vector4, ) diff --git a/fivefury/vector.py b/fivefury/vector.py index 621e43b..6593eba 100644 --- a/fivefury/vector.py +++ b/fivefury/vector.py @@ -3,6 +3,7 @@ import dataclasses import math from collections.abc import Callable, Iterable, Iterator +from typing import Self from . import _native_abi3 as _ffi @@ -22,6 +23,14 @@ def _components(value: Iterable[float], size: int, name: str) -> tuple[float, .. class _FloatValue: __slots__ = () + @classmethod + def from_rows(cls, rows: Iterable[Iterable[float]]) -> list[Self]: + """Materialize decoded component rows as independent nominal vectors.""" + if cls not in (Vector2, Vector3, Vector4, Quaternion): + raise TypeError("Batch construction requires a nominal vector class") + names = ("x", "y") if cls is Vector2 else ("x", "y", "z") if cls is Vector3 else ("x", "y", "z", "w") + return _ffi.vector_materialize(rows, cls, names) + @property def components(self) -> tuple[float, ...]: raise NotImplementedError @@ -617,6 +626,41 @@ def interpolate_vector4_many( ] +@dataclasses.dataclass(frozen=True, slots=True) +class _PointCloud3: + """Per-operation snapshot; bounds and packing share the same float64 data.""" + + data: bytes + + @classmethod + def from_points(cls, points: Iterable[Vector3]) -> _PointCloud3: + return cls(_ffi.vector_component_buffer(points, 3)) + + @property + def rows(self) -> memoryview: + return memoryview(self.data).cast("d", shape=(len(self.data) // 24, 3)) + + @property + def bounds(self) -> Aabb3: + if not self.data: + return Aabb3(Vector3(), Vector3()) + minimum, maximum = _ffi.bounds_from_vertices(self.rows) + return Aabb3(Vector3.from_iterable(minimum), Vector3.from_iterable(maximum)) + + def sphere_radius(self, center: Vector3) -> float: + if not self.data: + return 0.0 + return float(_ffi.bounds_sphere_radius_from_vertices(center.as_tuple(), self.rows)) + + def transformed(self, transform: object) -> _PointCloud3: + import numpy as np + + from .matrix import transform_position_array + + rows = np.frombuffer(self.data, dtype=np.float64).reshape((-1, 3)) + return _PointCloud3(transform_position_array(rows, transform).tobytes()) + + def sphere_radius_from_points(center: Vector3, points: Iterable[Vector3]) -> float: items = list(points) return ( diff --git a/fivefury/ydd/writer.py b/fivefury/ydd/writer.py index b177936..c4d8012 100644 --- a/fivefury/ydd/writer.py +++ b/fivefury/ydd/writer.py @@ -16,6 +16,7 @@ get_resource_total_page_count, layout_resource_sections, ) +from ..vector import Vector3 from ..ydr import YdrBuild from ..ydr.builder import ( _ROOT_SIZE, @@ -23,7 +24,12 @@ _write_drawable_payload, ) from ..ydr.gen9 import ShaderGen9Library, load_gen9_shader_library -from ..ydr.prepare import PreparedLods, PreparedMaterial, prepare_build +from ..ydr.prepare import ( + PreparedLods, + PreparedMaterial, + compute_model_collection_bounds, + prepare_build, +) from ..ydr.shaders import ShaderLibrary, load_shader_library from ..ydr.write_buffers import GraphicsWriter from ..ydr.write_drawable import pages_info_length, write_pages_info @@ -51,6 +57,7 @@ class _PreparedYddDrawable: materials: list[PreparedMaterial] lods: PreparedLods texture_sections: ResourceSections | None + geometry_bounds: tuple[Vector3, Vector3, Vector3, float] def _drawable_build(entry: YddDrawable, *, version: int) -> YdrBuild: @@ -103,6 +110,10 @@ def _prepare_ydd_drawables( materials=materials, lods=lods, texture_sections=_prepare_embedded_texture_dictionary(build, enhanced=enhanced), + geometry_bounds=compute_model_collection_bounds( + [model for models in lods.values() for model in models], + skeleton=build.skeleton, + ), ) ) return prepared @@ -164,6 +175,7 @@ def _build_ydd_payload( item.lods, page_counts, root_off=root_off, + geometry_bounds=item.geometry_bounds, texture_sections=item.texture_sections, drawable_file_vft=profile.drawable_headers.drawable, write_pages=False, diff --git a/fivefury/ydr/builder.py b/fivefury/ydr/builder.py index 2fbdfe8..e85f554 100644 --- a/fivefury/ydr/builder.py +++ b/fivefury/ydr/builder.py @@ -14,6 +14,7 @@ get_resource_total_page_count, layout_resource_sections, ) +from ..vector import Vector3 from .build_types import ( YdrBuild, YdrMaterialInput, @@ -166,6 +167,7 @@ def _write_drawable_payload( page_counts: tuple[int, int], *, root_off: int, + geometry_bounds: tuple[Vector3, Vector3, Vector3, float], texture_sections: ResourceSections | None, drawable_file_vft: int | None = None, write_pages: bool = True, @@ -263,10 +265,7 @@ def _write_model(model_off: int, lod_name: YdrLod, model_index: int) -> None: pages_info_off = system.alloc(pages_info_length(page_counts), 16) write_pages_info(system, pages_info_off, page_counts) - center, bounds_min, bounds_max, radius = compute_model_collection_bounds( - [model for lod_models in prepared_lods.values() for model in lod_models], - skeleton=source.skeleton, - ) + center, bounds_min, bounds_max, radius = geometry_bounds write_drawable_root( system, root_offset=root_off, @@ -317,6 +316,7 @@ def _build_system_payload( prepared_lods, page_counts: tuple[int, int], *, + geometry_bounds: tuple[Vector3, Vector3, Vector3, float], texture_sections: ResourceSections | None, recalculate_skeleton_hashes: bool = True, ) -> tuple[bytes, bytes, list[ResourceBlockSpan], list[ResourceBlockSpan]]: @@ -331,6 +331,7 @@ def _build_system_payload( prepared_lods, page_counts, root_off=0, + geometry_bounds=geometry_bounds, texture_sections=texture_sections, write_pages=True, recalculate_skeleton_hashes=recalculate_skeleton_hashes, @@ -395,6 +396,10 @@ def build_ydr_bytes( fill_vertex_colours=fill_vertex_colours, ) + geometry_bounds = compute_model_collection_bounds( + [model for models in prepared_lods.values() for model in models], + skeleton=source.skeleton, + ) texture_sections = _prepare_embedded_texture_dictionary(source, enhanced=enhanced) page_counts = (0, 0) system_data = b'' @@ -407,6 +412,7 @@ def build_ydr_bytes( prepared_materials, prepared_lods, page_counts, + geometry_bounds=geometry_bounds, texture_sections=texture_sections, recalculate_skeleton_hashes=recalculate_skeleton_hashes, ) @@ -436,6 +442,7 @@ def build_ydr_bytes( graphics_alignment=0x200, system_flags=system_flags, graphics_flags=graphics_flags, + compression_level=6, ) diff --git a/fivefury/ydr/prepare/bounds.py b/fivefury/ydr/prepare/bounds.py index b3268c3..c9cb896 100644 --- a/fivefury/ydr/prepare/bounds.py +++ b/fivefury/ydr/prepare/bounds.py @@ -2,8 +2,8 @@ from collections.abc import Sequence -from ...matrix import matrix4, transform_positions -from ...vector import Aabb3, Vector3, sphere_radius_from_points +from ...matrix import matrix4 +from ...vector import Vector3, _PointCloud3 from ..model.skeleton import YdrSkeleton from ..transforms import skeleton_absolute_transforms from .build import PreparedModel @@ -15,9 +15,10 @@ def compute_bounds( if not positions: zero = Vector3() return zero, zero, zero, 0.0 - bounds = Aabb3.from_points(positions) + points = _PointCloud3.from_points(positions) + bounds = points.bounds centre = bounds.center - radius = sphere_radius_from_points(centre, positions) + radius = points.sphere_radius(centre) return centre, bounds.minimum, bounds.maximum, radius @@ -28,6 +29,7 @@ def compute_model_collection_bounds( ) -> tuple[Vector3, Vector3, Vector3, float]: absolute_transforms = skeleton_absolute_transforms(skeleton) position_groups = [] + mesh_bounds = [] for model in models: transform = None binding = model.skeleton_binding @@ -40,22 +42,21 @@ def compute_model_collection_bounds( for mesh in model.meshes: if not mesh.positions: continue - position_groups.append( - mesh.positions + points = ( + mesh.points if transform is None # Serialized RAGE matrices act on row vectors; shared math uses columns. - else transform_positions(mesh.positions, matrix4(transform).T) + else mesh.points.transformed(matrix4(transform).T) ) + position_groups.append(points) + mesh_bounds.append(mesh.bounds if transform is None else points.bounds) if not position_groups: return compute_bounds(()) - mesh_bounds = [Aabb3.from_points(positions) for positions in position_groups] bounds = mesh_bounds[0] for mesh_bounds_value in mesh_bounds[1:]: bounds = bounds.merged(mesh_bounds_value) bb_min = bounds.minimum bb_max = bounds.maximum centre = bounds.center - radius = max( - sphere_radius_from_points(centre, positions) for positions in position_groups - ) + radius = max(points.sphere_radius(centre) for points in position_groups) return centre, bb_min, bb_max, radius diff --git a/fivefury/ydr/prepare/encoding.py b/fivefury/ydr/prepare/encoding.py index 80c39c3..a97594d 100644 --- a/fivefury/ydr/prepare/encoding.py +++ b/fivefury/ydr/prepare/encoding.py @@ -26,6 +26,7 @@ def _encode_vertex_bytes( colours1: Sequence[tuple[float, float, float, float]], blend_weights: Sequence[tuple[float, float, float, float]] | None = None, blend_indices: Sequence[tuple[int, int, int, int]] | None = None, + position_buffer: memoryview | None = None, ) -> tuple[int, int, int, bytes]: flags = 0 types_value = _DEFAULT_DECLARATION_TYPES @@ -49,6 +50,14 @@ def expand(values, semantic: VertexSemantic, *, fill: float = 0.0): arity = 3 case _: arity = 4 + nominal_arity = ( + 3 if semantic in (VertexSemantic.POSITION, VertexSemantic.NORMAL) + else 4 if semantic is VertexSemantic.TANGENT + else 2 if VertexSemantic.TEXCOORD0 <= semantic <= VertexSemantic.TEXCOORD7 + else None + ) + if arity == nominal_arity: + return values expanded = [] for value in values: components = tuple(value) @@ -75,7 +84,12 @@ def expand(values, semantic: VertexSemantic, *, fill: float = 0.0): (int(semantic), int(component_type)) for semantic, component_type in semantics ], - positions, + position_buffer + if ( + position_buffer is not None + and component_types.get(VertexSemantic.POSITION) == VertexComponentType.FLOAT3 + ) + else positions, normals, texcoords, tangents, @@ -98,6 +112,7 @@ def _encode_vertex_bytes_from_layout( *, blend_weights: Sequence[tuple[float, float, float, float]] | None = None, blend_indices: Sequence[tuple[int, int, int, int]] | None = None, + position_buffer: memoryview | None = None, ) -> tuple[int, int, int, bytes]: component_by_semantic: dict[VertexSemantic, VertexComponentType] = { semantic: component_type @@ -137,6 +152,7 @@ def _encode_vertex_bytes_from_layout( colours1, blend_weights=blend_weights, blend_indices=blend_indices, + position_buffer=position_buffer, ) @@ -152,6 +168,7 @@ def _encode_vertex_bytes_from_declaration( *, blend_weights: Sequence[tuple[float, float, float, float]] | None = None, blend_indices: Sequence[tuple[int, int, int, int]] | None = None, + position_buffer: memoryview | None = None, ) -> tuple[int, int, int, bytes]: component_types = { semantic: _canonical_component_type(semantic, component_type) @@ -174,4 +191,5 @@ def _encode_vertex_bytes_from_declaration( colours1, blend_weights=blend_weights, blend_indices=blend_indices, + position_buffer=position_buffer, ) diff --git a/fivefury/ydr/prepare/mesh.py b/fivefury/ydr/prepare/mesh.py index ed99d47..5c3d9a0 100644 --- a/fivefury/ydr/prepare/mesh.py +++ b/fivefury/ydr/prepare/mesh.py @@ -4,7 +4,7 @@ import struct from collections.abc import Mapping, Sequence -from ...vector import Vector2, Vector3, Vector4 +from ...vector import Aabb3, Vector2, Vector3, Vector4, _PointCloud3 from ..build_types import YdrMeshInput from ..model.skeleton import YdrSkeleton from ..shaders import ShaderLayoutDefinition @@ -37,6 +37,8 @@ class PreparedMesh: vertex_bytes: bytes index_bytes: bytes layout: ShaderLayoutDefinition + bounds: Aabb3 + points: _PointCloud3 def prepare_meshes( @@ -65,15 +67,17 @@ def prepare_meshes( skeleton=skeleton, ) for mesh in _split_mesh_by_vertex_limit(normalized): - positions = list(mesh.positions) - indices = list(mesh.indices) - normals = list(mesh.normals or ()) - texcoords = [list(channel) for channel in (mesh.texcoords or ())] - tangents = list(mesh.tangents or ()) - colours0 = list(mesh.colours0 or ()) - colours1 = list(mesh.colours1 or ()) - blend_weights = list(mesh.blend_weights or ()) - blend_indices = list(mesh.blend_indices or ()) + # Channel preparation and splitting own these lists for this save. + positions = mesh.positions + indices = mesh.indices + normals = mesh.normals or [] + texcoords = mesh.texcoords or [] + tangents = mesh.tangents or [] + colours0 = mesh.colours0 or [] + colours1 = mesh.colours1 or [] + blend_weights = mesh.blend_weights or [] + blend_indices = mesh.blend_indices or [] + points = _PointCloud3.from_points(positions) if ( mesh.declaration_flags is not None @@ -91,6 +95,7 @@ def prepare_meshes( colours1, blend_weights=blend_weights or None, blend_indices=blend_indices or None, + position_buffer=points.rows if positions else None, ) ) else: @@ -105,6 +110,7 @@ def prepare_meshes( colours1, blend_weights=blend_weights or None, blend_indices=blend_indices or None, + position_buffer=points.rows if positions else None, ) ) if max(indices, default=0) > 0xFFFF: @@ -133,6 +139,8 @@ def prepare_meshes( vertex_bytes=vertex_bytes, index_bytes=index_bytes, layout=layout, + bounds=points.bounds, + points=points, ) ) return prepared diff --git a/fivefury/ydr/prepare/splitting.py b/fivefury/ydr/prepare/splitting.py index 947f100..a84780c 100644 --- a/fivefury/ydr/prepare/splitting.py +++ b/fivefury/ydr/prepare/splitting.py @@ -1,11 +1,9 @@ from __future__ import annotations -import dataclasses from collections.abc import Sequence from typing import TypeVar from ... import _native as _native_backend -from ...mesh_math import triangle_array from ...vector import Vector2 from ..build_types import YdrMeshInput @@ -54,18 +52,19 @@ def _build_split_mesh( def _split_mesh_by_vertex_limit( mesh: YdrMeshInput, *, max_vertices: int = _MAX_MESH_UNIQUE_VERTICES ) -> list[YdrMeshInput]: - indices = triangle_array(mesh.indices, len(mesh.positions)).reshape(-1).tolist() - normalized = dataclasses.replace(mesh, indices=indices) + # prepare_channels already normalized the topology. The native splitter + # validates its own index boundary without rebuilding every vector channel. + indices = mesh.indices if not indices: - return [normalized] + return [mesh] chunks = _native_backend._ydr_split_mesh_indices( indices, len(mesh.positions), max_vertices, ) if chunks is None: - return [normalized] + return [mesh] return [ - _build_split_mesh(normalized, vertex_indices, remapped_indices) + _build_split_mesh(mesh, vertex_indices, remapped_indices) for vertex_indices, remapped_indices in chunks ] diff --git a/fivefury/ydr/reader.py b/fivefury/ydr/reader.py index 773f226..ab12deb 100644 --- a/fivefury/ydr/reader.py +++ b/fivefury/ydr/reader.py @@ -23,7 +23,6 @@ split_rsc7_sections, virtual_to_offset, ) -from ..vector import Vector2, Vector3, Vector4 from ..ytd import Ytd, read_embedded_texture_dictionary from .defs import ( DAT_PHYSICAL_BASE, @@ -219,13 +218,10 @@ def _parse_mesh( material_index=material_index, material=material, indices=indices, - positions=[Vector3.from_iterable(value) for value in decoded["positions"]], - normals=[Vector3.from_iterable(value) for value in decoded["normals"]], - tangents=[Vector4.from_iterable(value) for value in decoded["tangents"]], - texcoords=[ - [Vector2.from_iterable(value) for value in channel] - for channel in decoded["texcoords"] - ], + positions=decoded["positions"], + normals=decoded["normals"], + tangents=decoded["tangents"], + texcoords=decoded["texcoords"], colours0=list(decoded["colours0"]), colours1=list(decoded["colours1"]), blend_weights=list(decoded["blend_weights"]), diff --git a/fivefury/ydr/write_buffers.py b/fivefury/ydr/write_buffers.py index 9a77d13..26e10ad 100644 --- a/fivefury/ydr/write_buffers.py +++ b/fivefury/ydr/write_buffers.py @@ -13,7 +13,7 @@ build_gen9_vertex_declaration, build_shader_resource_view_g9, ) -from .prepare import PreparedMesh, compute_bounds +from .prepare import PreparedMesh @dataclasses.dataclass(slots=True) @@ -162,11 +162,10 @@ def build_mesh_buffer_pack( system.pack_into('I', index_buffer_off + 0x0C, 0) system.pack_into('Q', index_buffer_off + 0x10, virtual(index_data_off) if index_data_off else 0) - _center, bounds_min, bounds_max, _radius = compute_bounds(mesh.positions) return MeshBufferPack( mesh=mesh, - bounds_min=bounds_min, - bounds_max=bounds_max, + bounds_min=mesh.bounds.minimum, + bounds_max=mesh.bounds.maximum, declaration_off=declaration_off, vertex_data_off=vertex_data_off, index_data_off=index_data_off, diff --git a/fivefury/yft/writer.py b/fivefury/yft/writer.py index a3d9500..a12f662 100644 --- a/fivefury/yft/writer.py +++ b/fivefury/yft/writer.py @@ -20,7 +20,7 @@ get_resource_total_page_count, layout_resource_sections, ) -from ..vector import Vector4 +from ..vector import Vector3, Vector4 from ..ydr import Ydr, YdrBuild, YdrLight from ..ydr.builder import _prepare_embedded_texture_dictionary, _write_drawable_payload from ..ydr.gen9 import load_gen9_shader_library @@ -89,6 +89,7 @@ class _PreparedFragmentDrawable: build: YdrBuild materials: list[PreparedMaterial] lods: PreparedLods + geometry_bounds: tuple[Vector3, Vector3, Vector3, float] texture_sections: ResourceSections | None = None fragment: YftFragmentDrawable | YftFragmentDrawableBuild | None = None source_id: int = 0 @@ -153,6 +154,10 @@ def _prepare_drawable( build=build, materials=materials, lods=lods, + geometry_bounds=compute_model_collection_bounds( + [model for models in lods.values() for model in models], + skeleton=build.skeleton, + ), fragment=( drawable if isinstance(drawable, (YftFragmentDrawable, YftFragmentDrawableBuild)) @@ -724,6 +729,7 @@ def _build_yft_payload( item.lods, page_counts, root_off=item.root_offset, + geometry_bounds=item.geometry_bounds, texture_sections=item.texture_sections, runtime_headers=runtime_headers.drawable, enhanced=runtime_headers.enhanced, @@ -828,10 +834,7 @@ def build_yft_bytes( if prepared[0] is None: raise ValueError("YFT writer requires a common drawable") if source.bounding_sphere == Vector4(): - _center, _bounds_min, _bounds_max, radius = compute_model_collection_bounds( - [model for lods in prepared[0].lods.values() for model in lods], - skeleton=prepared[0].build.skeleton, - ) + _center, _bounds_min, _bounds_max, radius = prepared[0].geometry_bounds source = dataclasses.replace( source, bounding_sphere=Vector4(_center.x, _center.y, _center.z, radius), diff --git a/native/bounds/python_conversions.cpp b/native/bounds/python_conversions.cpp index 808c342..e6a126a 100644 --- a/native/bounds/python_conversions.cpp +++ b/native/bounds/python_conversions.cpp @@ -1,4 +1,5 @@ #include "bounds/python_conversions.h" +#include "python/float_rows.h" #include #include @@ -45,6 +46,19 @@ bool parse_vector3(PyObject* object, fivefury_native::bounds::Vec3& out, const c } bool parse_vertices(PyObject* object, std::vector& out, const char* argument_name) { + if (PyObject_CheckBuffer(object)) { + FloatRows rows; + if (!rows.acquire(object, 3)) return false; + out.resize(static_cast(rows.buffer.shape[0])); + for (Py_ssize_t index = 0; index < rows.buffer.shape[0]; ++index) { + out[index] = { + fivefury_native::bounds::canonical_zero(rows.value(index, 0)), + fivefury_native::bounds::canonical_zero(rows.value(index, 1)), + fivefury_native::bounds::canonical_zero(rows.value(index, 2)), + }; + } + return true; + } PyHandle sequence_owner(PySequence_Fast(object, argument_name)); PyObject* sequence = sequence_owner.get(); if (sequence == nullptr) { diff --git a/native/drawable/bindings.h b/native/drawable/bindings.h index fd19d0b..41bad0f 100644 --- a/native/drawable/bindings.h +++ b/native/drawable/bindings.h @@ -3,6 +3,8 @@ #include "python/bridge.h" namespace fivefury_py { +PyObject* mod_vector_materialize(PyObject*, PyObject*); +PyObject* mod_vector_component_buffer(PyObject*, PyObject*); PyObject* mod_ydr_pack_vertex_buffer(PyObject*, PyObject* args); PyObject* mod_ydr_decode_vertex_buffer(PyObject*, PyObject* args); diff --git a/native/drawable/vector_bindings.cpp b/native/drawable/vector_bindings.cpp index 3c342b5..617141a 100644 --- a/native/drawable/vector_bindings.cpp +++ b/native/drawable/vector_bindings.cpp @@ -1,11 +1,77 @@ #include "drawable/bindings.h" #include "math/vector.h" +#include "python/vector_factory.h" #include +#include namespace fivefury_py { +PyObject* mod_vector_component_buffer(PyObject*, PyObject* args) { + PyObject* values; + int dimensions; + if (!PyArg_ParseTuple(args, "Oi", &values, &dimensions)) return nullptr; + if (dimensions < 2 || dimensions > 4) { + PyErr_SetString(PyExc_ValueError, "Vectors require 2 to 4 components"); return nullptr; + } + PyHandle sequence(PySequence_Fast(values, "vectors must be a sequence")); + if (!sequence) return nullptr; + const auto count = PySequence_Size(sequence.get()); + if (count < 0) return nullptr; + PyHandle result(PyBytes_FromStringAndSize(nullptr, checked_buffer_size(count, dimensions * sizeof(double)))); + if (!result) return nullptr; + char* output = PyBytes_AsString(result.get()); + if (!output) return nullptr; + const char* names[] = {"x", "y", "z", "w"}; + for (Py_ssize_t row = 0; row < count; ++row) { + PyHandle value(PySequence_GetItem(sequence.get(), row)); + if (!value) return nullptr; + for (int column = 0; column < dimensions; ++column) { + PyHandle component(PyObject_GetAttrString(value.get(), names[column])); + if (!component) return nullptr; + const double number = PyFloat_AsDouble(component.get()); + if (PyErr_Occurred()) return nullptr; + std::memcpy(output + (row * dimensions + column) * sizeof(double), &number, sizeof(number)); + } + } + return result.release(); +} + +PyObject* mod_vector_materialize(PyObject*, PyObject* args) { + PyObject *rows, *cls, *names; + if (!PyArg_ParseTuple(args, "OOO", &rows, &cls, &names)) return nullptr; + PyHandle fields(PySequence_Tuple(names)); + PyHandle sequence(PySequence_Tuple(rows)); + if (!fields || !sequence) return nullptr; + const auto dimensions = PyTuple_Size(fields.get()); + if (dimensions < 2 || dimensions > 4) { + PyErr_SetString(PyExc_ValueError, "Vectors require 2 to 4 components"); return nullptr; + } + VectorFactory factory(cls, fields.get()); + if (!factory) return nullptr; + const auto count = PyTuple_Size(sequence.get()); + PyHandle result(PyList_New(count)); + if (!result) return nullptr; + for (Py_ssize_t i = 0; i < count; ++i) { + PyHandle components(PySequence_Tuple(PyTuple_GetItem(sequence.get(), i))); + if (!components) return nullptr; + if (PyTuple_Size(components.get()) != dimensions) { + PyErr_SetString(PyExc_ValueError, "Incorrect vector component count"); return nullptr; + } + // The Python boundary supplies the nominal vector class. Match its + // float coercion while filling frozen slots before exposing the object. + PyHandle object(factory.create()); + if (!object) return nullptr; + for (Py_ssize_t j = 0; j < dimensions; ++j) { + PyHandle value(PyNumber_Float(PyTuple_GetItem(components.get(), j))); + if (!factory.assign(object.get(), j, value.get())) return nullptr; + } + if (!list_take(result.get(), i, object.release())) return nullptr; + } + return result.release(); +} + namespace { bool parse_vec4_sequence(PyObject* object, std::vector& out) { diff --git a/native/drawable/vertex_bindings.cpp b/native/drawable/vertex_bindings.cpp index c87b3e3..cee2a84 100644 --- a/native/drawable/vertex_bindings.cpp +++ b/native/drawable/vertex_bindings.cpp @@ -1,4 +1,5 @@ #include "drawable/bindings.h" +#include "python/float_rows.h" #include #include @@ -227,6 +228,17 @@ bool flatten_channel( if (arity == 0) { return true; } + if (PyObject_CheckBuffer(channel)) { + FloatRows rows; + if (!rows.acquire(channel, static_cast(arity))) return false; + if (static_cast(rows.buffer.shape[0]) < vertex_count) { + PyErr_Format(PyExc_ValueError, "vertex channel '%s' is shorter than the vertex count", channel_name); + return false; + } + out.resize(vertex_count * arity); + if (!out.empty()) std::memcpy(out.data(), rows.buffer.buf, out.size() * sizeof(double)); + return true; + } PyHandle sequence_owner(PySequence_Fast(channel, "vertex channel must be a sequence")); PyObject* sequence = sequence_owner.get(); if (sequence == nullptr) { @@ -493,7 +505,11 @@ PyObject* mod_ydr_pack_vertex_buffer(PyObject*, PyObject* args) { } try { - const auto buffer = pack_buffer(specs, vertex_count); + std::string buffer; + { + GilRelease release; + buffer = pack_buffer(specs, vertex_count); + } return PyBytes_FromStringAndSize(buffer.data(), static_cast(buffer.size())); } catch (...) { return translate_cpp_exception(); diff --git a/native/drawable/vertex_decode.cpp b/native/drawable/vertex_decode.cpp index 973da6a..59c4c59 100644 --- a/native/drawable/vertex_decode.cpp +++ b/native/drawable/vertex_decode.cpp @@ -1,4 +1,5 @@ #include "drawable/bindings.h" +#include "python/vector_factory.h" #include #include @@ -221,19 +222,21 @@ bool parse_indices(PyObject* object, std::vector& out) { PyObject* mod_ydr_decode_vertex_buffer(PyObject*, PyObject* args) { PyObject* data_object = nullptr; PyObject* offsets_object = nullptr; + PyObject *vector2_type, *vector3_type, *vector4_type; Py_ssize_t vertex_count = 0; int stride = 0; unsigned long long flags = 0; unsigned long long types = 0; if (!PyArg_ParseTuple( args, - "OniKKO", + "OniKKOOOO", &data_object, &vertex_count, &stride, &flags, &types, - &offsets_object + &offsets_object, + &vector2_type, &vector3_type, &vector4_type )) { return nullptr; } @@ -279,83 +282,112 @@ PyObject* mod_ydr_decode_vertex_buffer(PyObject*, PyObject* args) { const auto data_size = static_cast(buffer.len); DecodedVertices decoded; { - GilRelease gil_release; - for (Py_ssize_t vertex = 0; vertex < count; ++vertex) { - const auto base = static_cast(vertex) * static_cast(stride); + GilRelease gil_release; + // Resolve each declaration once, then walk its channel. Per-channel order + // is unchanged, including partial buffers and explicit Gen9 offsets. for (int semantic = 0; semantic < 16; ++semantic) { if (((flags >> semantic) & 1UL) == 0UL) { continue; } const auto type = static_cast((types >> (semantic * 4)) & 0xFULL); const auto size = component_size(type); - const auto source = base + static_cast(offsets[semantic]); - if (size <= 0 || source + static_cast(size) > data_size) { - continue; - } - if (semantic == SEMANTIC_BLEND_INDICES && size == 4) { - decoded.blend_indices.push_back({ - data[source + 2], data[source + 1], data[source], data[source + 3] - }); + if (size <= 0 || offsets[semantic] < 0 || + static_cast(offsets[semantic]) > data_size || + static_cast(size) > data_size - static_cast(offsets[semantic])) { continue; } - if (semantic == SEMANTIC_BLEND_WEIGHTS && type == COMPONENT_COLOUR) { - decoded.blend_weights.push_back({ - static_cast(data[source + 2]) / 255.0, - static_cast(data[source + 1]) / 255.0, - static_cast(data[source]) / 255.0, - static_cast(data[source + 3]) / 255.0, - }); - continue; - } - const auto value = decode_component(data + source, type); - if (value.count == 0) { - continue; + const auto offset = static_cast(offsets[semantic]); + const auto channel_count = std::min( + static_cast(count), + (data_size - offset - size) / static_cast(stride) + 1U + ); + switch (semantic) { + case SEMANTIC_POSITION: decoded.positions.reserve(channel_count); break; + case SEMANTIC_NORMAL: decoded.normals.reserve(channel_count); break; + case SEMANTIC_TANGENT: decoded.tangents.reserve(channel_count); break; + case SEMANTIC_COLOUR0: decoded.colours0.reserve(channel_count); break; + case SEMANTIC_COLOUR1: decoded.colours1.reserve(channel_count); break; + case SEMANTIC_BLEND_WEIGHTS: decoded.blend_weights.reserve(channel_count); break; + case SEMANTIC_BLEND_INDICES: decoded.blend_indices.reserve(channel_count); break; + default: + if (semantic >= SEMANTIC_TEXCOORD0 && semantic <= SEMANTIC_TEXCOORD7) + decoded.texcoords[semantic - SEMANTIC_TEXCOORD0].reserve(channel_count); } - if (semantic == SEMANTIC_POSITION && value.count >= 3) { - decoded.positions.push_back({value.values[0], value.values[1], value.values[2]}); - } else if (semantic == SEMANTIC_NORMAL && value.count >= 3) { - decoded.normals.push_back({value.values[0], value.values[1], value.values[2]}); - } else if (semantic == SEMANTIC_TANGENT && value.count >= 4) { - decoded.tangents.push_back(value.values); - } else if (semantic == SEMANTIC_COLOUR0 && value.count >= 4) { - decoded.colours0.push_back(value.values); - } else if (semantic == SEMANTIC_COLOUR1 && value.count >= 4) { - decoded.colours1.push_back(value.values); - } else if (semantic == SEMANTIC_BLEND_INDICES && value.count >= 4) { - decoded.blend_indices.push_back({ - static_cast(value.values[0]), - static_cast(value.values[1]), - static_cast(value.values[2]), - static_cast(value.values[3]), - }); - } else if (semantic == SEMANTIC_BLEND_WEIGHTS && value.count >= 4) { - auto weights = value.values; - if (value.integral) { - for (auto& component : weights) { - component /= 255.0; + for (std::size_t vertex = 0; vertex < channel_count; ++vertex) { + const auto source = vertex * static_cast(stride) + offset; + if (semantic == SEMANTIC_BLEND_INDICES && size == 4) { + decoded.blend_indices.push_back({ + data[source + 2], data[source + 1], data[source], data[source + 3] + }); + continue; + } + if (semantic == SEMANTIC_BLEND_WEIGHTS && type == COMPONENT_COLOUR) { + decoded.blend_weights.push_back({ + static_cast(data[source + 2]) / 255.0, + static_cast(data[source + 1]) / 255.0, + static_cast(data[source]) / 255.0, + static_cast(data[source + 3]) / 255.0, + }); + continue; + } + const auto value = decode_component(data + source, type); + if (value.count == 0) { + continue; + } + if (semantic == SEMANTIC_POSITION && value.count >= 3) { + decoded.positions.push_back({value.values[0], value.values[1], value.values[2]}); + } else if (semantic == SEMANTIC_NORMAL && value.count >= 3) { + decoded.normals.push_back({value.values[0], value.values[1], value.values[2]}); + } else if (semantic == SEMANTIC_TANGENT && value.count >= 4) { + decoded.tangents.push_back(value.values); + } else if (semantic == SEMANTIC_COLOUR0 && value.count >= 4) { + decoded.colours0.push_back(value.values); + } else if (semantic == SEMANTIC_COLOUR1 && value.count >= 4) { + decoded.colours1.push_back(value.values); + } else if (semantic == SEMANTIC_BLEND_INDICES && value.count >= 4) { + decoded.blend_indices.push_back({ + static_cast(value.values[0]), + static_cast(value.values[1]), + static_cast(value.values[2]), + static_cast(value.values[3]), + }); + } else if (semantic == SEMANTIC_BLEND_WEIGHTS && value.count >= 4) { + auto weights = value.values; + if (value.integral) { + for (auto& component : weights) { + component /= 255.0; + } } + decoded.blend_weights.push_back(weights); + } else if ( + semantic >= SEMANTIC_TEXCOORD0 && semantic <= SEMANTIC_TEXCOORD7 && + value.count >= 2 + ) { + const auto index = semantic - SEMANTIC_TEXCOORD0; + decoded.max_texcoord = std::max(decoded.max_texcoord, index); + decoded.texcoords[static_cast(index)].push_back({ + value.values[0], value.values[1] + }); } - decoded.blend_weights.push_back(weights); - } else if ( - semantic >= SEMANTIC_TEXCOORD0 && semantic <= SEMANTIC_TEXCOORD7 && - value.count >= 2 - ) { - const auto index = semantic - SEMANTIC_TEXCOORD0; - decoded.max_texcoord = std::max(decoded.max_texcoord, index); - decoded.texcoords[static_cast(index)].push_back({ - value.values[0], value.values[1] - }); } } } - } buffer.release(); + PyHandle names2(Py_BuildValue("(ss)", "x", "y")); + PyHandle names3(Py_BuildValue("(sss)", "x", "y", "z")); + PyHandle names4(Py_BuildValue("(ssss)", "x", "y", "z", "w")); + if (!names2 || !names3 || !names4) return nullptr; + VectorFactory vectors2(vector2_type, names2.get()); + VectorFactory vectors3(vector3_type, names3.get()); + VectorFactory vectors4(vector4_type, names4.get()); + if (!vectors2 || !vectors3 || !vectors4) return nullptr; + PyObject* result = PyDict_New(); if (result == nullptr || - !dict_set_owned(result, "positions", build_float_tuples(decoded.positions)) || - !dict_set_owned(result, "normals", build_float_tuples(decoded.normals)) || - !dict_set_owned(result, "tangents", build_float_tuples(decoded.tangents)) || + !dict_set_owned(result, "positions", vectors3.materialize(decoded.positions)) || + !dict_set_owned(result, "normals", vectors3.materialize(decoded.normals)) || + !dict_set_owned(result, "tangents", vectors4.materialize(decoded.tangents)) || !dict_set_owned(result, "colours0", build_float_tuples(decoded.colours0)) || !dict_set_owned(result, "colours1", build_float_tuples(decoded.colours1)) || !dict_set_owned(result, "blend_weights", build_float_tuples(decoded.blend_weights)) || @@ -369,7 +401,9 @@ PyObject* mod_ydr_decode_vertex_buffer(PyObject*, PyObject* args) { return nullptr; } for (int index = 0; index <= decoded.max_texcoord; ++index) { - PyList_SetItem(texcoords, index, build_float_tuples(decoded.texcoords[index])); + if (!list_take(texcoords, index, vectors2.materialize(decoded.texcoords[index]))) { + Py_DECREF(texcoords); Py_DECREF(result); return nullptr; + } } if (!dict_set_owned(result, "texcoords", texcoords)) { Py_DECREF(result); diff --git a/native/python/float_rows.h b/native/python/float_rows.h new file mode 100644 index 0000000..d188bc5 --- /dev/null +++ b/native/python/float_rows.h @@ -0,0 +1,32 @@ +#pragma once + +#include "python/bridge.h" +#include + +namespace fivefury_py { + +// Checked, potentially unaligned float64 rows at the Python buffer boundary. +struct FloatRows { + Buffer buffer; + + bool acquire(PyObject* object, Py_ssize_t width) { + if (PyObject_GetBuffer(object, &buffer, PyBUF_FORMAT | PyBUF_ND | PyBUF_STRIDES) < 0) + return false; + if (buffer.ndim != 2 || buffer.itemsize != sizeof(double) || + !buffer.format || std::strcmp(buffer.format, "d") != 0 || + buffer.shape[1] != width || !PyBuffer_IsContiguous(&buffer, 'C')) { + PyErr_SetString(PyExc_ValueError, "expected contiguous float64 rows with matching component count"); + return false; + } + return true; + } + + double value(Py_ssize_t row, Py_ssize_t column) const { + double result; + const auto offset = (row * buffer.shape[1] + column) * sizeof(double); + std::memcpy(&result, static_cast(buffer.buf) + offset, sizeof(result)); + return result; + } +}; + +} // namespace fivefury_py diff --git a/native/python/module.cpp b/native/python/module.cpp index 5e6635d..6e81230 100644 --- a/native/python/module.cpp +++ b/native/python/module.cpp @@ -11,6 +11,8 @@ namespace fivefury_py { PyMethodDef module_methods[] = { + {"vector_materialize", guarded_call, METH_VARARGS, nullptr}, + {"vector_component_buffer", guarded_call, METH_VARARGS, nullptr}, {"meta_scalars_new", guarded_call, METH_VARARGS, nullptr}, {"meta_scalars_read", guarded_call, METH_VARARGS, nullptr}, {"meta_scalars_write", guarded_call, METH_VARARGS, nullptr}, diff --git a/native/python/vector_factory.h b/native/python/vector_factory.h new file mode 100644 index 0000000..ff64c4f --- /dev/null +++ b/native/python/vector_factory.h @@ -0,0 +1,46 @@ +#pragma once + +#include "python/bridge.h" +#include +#include + +namespace fivefury_py { + +// Fill nominal frozen-vector slots before exposing a new object to Python. +// Both decoded native arrays and Python component rows use this boundary. +class VectorFactory { + PyObject* type_; + PyHandle names_; + PyHandle allocate_; +public: + VectorFactory(PyObject* type, PyObject* names) + : type_(type), names_(PySequence_Tuple(names)), + allocate_(PyObject_GetAttrString(reinterpret_cast(&PyBaseObject_Type), "__new__")) {} + explicit operator bool() const { return names_ && allocate_; } + Py_ssize_t dimensions() const { return PyTuple_Size(names_.get()); } + PyObject* create() const { + return PyObject_CallFunctionObjArgs(allocate_.get(), type_, nullptr); + } + bool assign(PyObject* object, Py_ssize_t index, PyObject* value) const { + return value && PyObject_GenericSetAttr(object, PyTuple_GetItem(names_.get(), index), value) == 0; + } + template + PyObject* materialize(const std::vector>& rows) const { + if (dimensions() != N) { + PyErr_SetString(PyExc_ValueError, "Incorrect vector component count"); return nullptr; + } + PyHandle result(PyList_New(static_cast(rows.size()))); + if (!result) return nullptr; + for (std::size_t i = 0; i < rows.size(); ++i) { + PyHandle object(create()); + if (!object) return nullptr; + for (std::size_t j = 0; j < N; ++j) { + PyHandle value(PyFloat_FromDouble(rows[i][j])); + if (!assign(object.get(), j, value.get())) return nullptr; + } + if (!list_take(result.get(), i, object.release())) return nullptr; + } + return result.release(); + } +}; +} diff --git a/tests/drawable/ydr/test_ydr_prepared_bounds.py b/tests/drawable/ydr/test_ydr_prepared_bounds.py index ad2352f..1b30c1e 100644 --- a/tests/drawable/ydr/test_ydr_prepared_bounds.py +++ b/tests/drawable/ydr/test_ydr_prepared_bounds.py @@ -8,16 +8,18 @@ import pytest from fivefury import Vector2, Vector3 +from fivefury import matrix as matrix_math from fivefury.resource import split_rsc7_sections +from fivefury.vector import _PointCloud3 from fivefury.ydr import ( YdrBone, YdrMeshInput, YdrSkeleton, YdrSkeletonBinding, + builder, create_ydr, ) from fivefury.ydr.prepare import ( - bounds, compute_bounds, compute_model_collection_bounds, prepare_build, @@ -77,6 +79,33 @@ def test_transformed_rigid_bounds_match_expected_float32_bytes(version: int) -> assert build.skeleton.transformations == [matrix] +@pytest.mark.parametrize("version", [165, 159]) +def test_geometry_is_prepared_once_per_save_and_refreshes_after_edits(version): + mesh = YdrMeshInput( + positions=[Vector3(), Vector3(1, 0, 0), Vector3(0, 1, 0)], + indices=[0, 1, 2], + normals=[Vector3(0, 0, 1)] * 3, + texcoords=[[Vector2(), Vector2(1, 0), Vector2(0, 1)]], + ) + build = create_ydr(meshes=[mesh], version=version) + before = copy.deepcopy(build) + with ( + patch.object(_PointCloud3, "from_points", wraps=_PointCloud3.from_points) as points, + patch.object(builder, "compute_model_collection_bounds", wraps=builder.compute_model_collection_bounds) as root_bounds, + patch.object(builder, "_build_system_payload", wraps=builder._build_system_payload) as payload, + ): + first = build.to_bytes() + assert points.call_count == root_bounds.call_count == 1 + assert payload.call_count >= 2 + assert build == before + mesh.positions[1] = Vector3(9, 0, 0) + second = build.to_bytes() + assert points.call_count == root_bounds.call_count == 2 + assert first != second + _, system, _ = split_rsc7_sections(second) + assert struct.unpack_from("<3f", system, 0x40) == (9, 1, 0) + + @pytest.mark.parametrize( "matrix", [ @@ -132,7 +161,7 @@ def test_row_adapter_numerical_parity_and_input_preservation(matrix) -> None: for point in models[0].meshes[0].positions ] with patch.object( - bounds, "transform_positions", wraps=bounds.transform_positions + matrix_math, "transform_position_array", wraps=matrix_math.transform_position_array ) as transform: actual = compute_model_collection_bounds(models, skeleton=build.skeleton) expected = compute_bounds(expected_positions) diff --git a/tests/runtime/test_point_buffers.py b/tests/runtime/test_point_buffers.py new file mode 100644 index 0000000..06c50fc --- /dev/null +++ b/tests/runtime/test_point_buffers.py @@ -0,0 +1,94 @@ +import copy +import math +import struct +from unittest.mock import patch + +import numpy as np +import pytest + +from fivefury import Vector3 +from fivefury import _native_abi3 as native +from fivefury._native import _ydr_decode_vertex_buffer +from fivefury.vector import Aabb3, _PointCloud3, sphere_radius_from_points + + +def test_point_snapshot_reuses_float64_values_without_quantizing_bounds(): + values = [Vector3(-0.0, 0.123456789, 2), Vector3(4, -5, 6)] + cloud = _PointCloud3.from_points(iter(values)) + before = copy.deepcopy(cloud) + assert cloud.rows.readonly + assert cloud.rows.shape == (2, 3) + assert cloud.bounds == Aabb3.from_points(values) + assert cloud.sphere_radius(cloud.bounds.center) == sphere_radius_from_points( + cloud.bounds.center, values + ) + assert math.copysign(1, struct.unpack_from("=d", cloud.data)[0]) == -1 + packed = native.ydr_pack_vertex_buffer( + [(0, 6)], cloud.rows, [], [], [], [], [] + ) + assert packed == struct.pack("<6f", *(component for value in values for component in value)) + values[0] = Vector3(99, 99, 99) + assert cloud == before + assert cloud.bounds.maximum == Vector3(4, .123456789, 6) + + +def test_empty_point_snapshot(): + cloud = _PointCloud3.from_points([]) + assert cloud.bounds == Aabb3(Vector3(), Vector3()) + assert cloud.sphere_radius(Vector3(99, 99, 99)) == 0 + + +@pytest.mark.parametrize("dimensions", [0, 1, 5, -1]) +def test_component_buffer_rejects_unsupported_dimensions(dimensions): + with pytest.raises(ValueError, match="2 to 4"): + native.vector_component_buffer([Vector3()], dimensions) + + +def test_component_buffer_rejects_missing_nominal_components(): + with pytest.raises(AttributeError): + native.vector_component_buffer([(1, 2, 3)], 3) + + +def test_point_transform_keeps_coordinates_in_buffers(): + cloud = _PointCloud3.from_points([Vector3(1, 2, 3)]) + transform = ((2, 0, 0, 10), (0, -3, 0, 20), (0, 0, 4, 30), (0, 0, 0, 1)) + with patch.object(native, "vector_component_buffer", side_effect=AssertionError("reconverted vectors")): + result = cloud.transformed(transform) + empty = _PointCloud3(b"").transformed(transform) + assert struct.unpack("=3d", result.data) == (12, 14, 42) + assert struct.unpack("=3d", cloud.data) == (1, 2, 3) + assert empty.data == b"" + + +@pytest.mark.parametrize("rows", [ + np.zeros((2, 3), dtype=np.float32), + np.zeros((2, 4), dtype=np.float64), + np.zeros((4, 3), dtype=np.float64)[::2], + np.zeros((2, 3), dtype=">f8"), +]) +def test_native_float64_buffer_boundary_rejects_incompatible_views(rows): + with pytest.raises(ValueError, match="contiguous float64"): + native.bounds_from_vertices(memoryview(rows)) + with pytest.raises(ValueError, match="contiguous float64"): + native.ydr_pack_vertex_buffer([(0, 6)], memoryview(rows), [], [], [], [], []) + + +def test_native_float64_rows_allow_unaligned_storage(): + data = b"x" + struct.pack("=3d", 1, 2, 3) + rows = memoryview(data)[1:].cast("d", shape=(1, 3)) + assert native.bounds_from_vertices(rows) == ((1, 2, 3), (1, 2, 3)) + assert native.ydr_pack_vertex_buffer([(0, 6)], rows, [], [], [], [], []) == struct.pack("<3f", 1, 2, 3) + + +def test_decoder_channel_walk_preserves_explicit_offsets_and_partial_buffers(): + offsets = [0] * 16 + offsets[0] = 8 + raw = struct.pack("<10f", .25, .75, 1, 2, 3, .5, 1, 4, 5, 6) + decoded = _ydr_decode_vertex_buffer(raw, 99, 20, 0x41, 6 | (5 << 24), tuple(offsets)) + assert decoded["positions"] == [Vector3(1, 2, 3), Vector3(4, 5, 6)] + assert [uv.as_tuple() for uv in decoded["texcoords"][0]] == [(.25, .75), (.5, 1)] + offsets[0] = len(raw) - 12 + decoded = _ydr_decode_vertex_buffer(raw, 99, 20, 1, 6, tuple(offsets)) + assert decoded["positions"] == [Vector3(4, 5, 6)] + offsets[0] = -1 + assert _ydr_decode_vertex_buffer(raw, 99, 20, 1, 6, tuple(offsets))["positions"] == [] diff --git a/tests/runtime/test_vector_materialize.py b/tests/runtime/test_vector_materialize.py new file mode 100644 index 0000000..eb9562e --- /dev/null +++ b/tests/runtime/test_vector_materialize.py @@ -0,0 +1,63 @@ +import dataclasses +import math +import pickle +import struct +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from fivefury import Quaternion, Vector2, Vector3, Vector4 +from fivefury import _native_abi3 as native +from fivefury.ydr.reader import _decode_vertices + + +@pytest.mark.parametrize("cls,rows", [ + (Vector2, [(1, "2"), (-0.0, 3.5)]), + (Vector3, [(1, 2, 3), (-0.0, 0, 4)]), + (Vector4, [(1, 2, 3, 4)]), + (Quaternion, [(0, 0, 0, 1)]), +]) +def test_batch_vectors_match_constructors(cls, rows): + result = cls.from_rows(iter(rows)) + assert result == [cls.from_iterable(row) for row in rows] + assert all(type(value) is cls for value in result) + assert pickle.loads(pickle.dumps(result)) == result + assert cls.from_rows([]) == [] + with pytest.raises(dataclasses.FrozenInstanceError): + result[0].x = 7 + + +def test_batch_vectors_do_not_share_instances_and_preserve_negative_zero(): + left, right = Vector3.from_rows([(-0.0, 1, 2)] * 2) + assert left is not right + assert math.copysign(1, left.x) == -1 + + +@pytest.mark.parametrize("rows,error", [([(1, 2)], ValueError), ([(1, 2, 3, 4)], ValueError), ([(1, object(), 3)], TypeError)]) +def test_batch_vectors_reject_invalid_components(rows, error): + with pytest.raises(error): + Vector3.from_rows(rows) + + +def test_vertex_decoder_emits_nominal_channels_directly(): + data = struct.pack("<3f3f2f4f", 1, 2, 3, 0, 0, 1, .25, .75, 1, 0, 0, -1) + flags = (1 << 0) | (1 << 3) | (1 << 6) | (1 << 14) + types = 6 | (6 << 12) | (5 << 24) | (7 << 56) + decoded = _decode_vertices(data * 2, 2, len(data), flags, types) + assert decoded["positions"] == [Vector3(1, 2, 3)] * 2 + assert decoded["normals"] == [Vector3(0, 0, 1)] * 2 + assert decoded["texcoords"] == [[Vector2(.25, .75)] * 2] + assert decoded["tangents"] == [Vector4(1, 0, 0, -1)] * 2 + assert decoded["positions"][0] is not decoded["positions"][1] + with pytest.raises(dataclasses.FrozenInstanceError): + decoded["positions"][0].x = 7 + + +def test_vertex_packing_parallel_calls_preserve_independent_outputs(): + def pack(index): + return native.ydr_pack_vertex_buffer( + [(0, 6)], [Vector3(index, 2, 3)], [], [], [], [], [], None, None + ) + with ThreadPoolExecutor(max_workers=4) as pool: + outputs = list(pool.map(pack, range(16))) + assert outputs == [struct.pack("<3f", i, 2, 3) for i in range(16)]