diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e46e81..61308e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,5 +35,49 @@ jobs: cache: npm cache-dependency-path: tools/bfme-launcher-mcp/package-lock.json - run: npm.cmd ci + - run: npm.cmd audit --audit-level=moderate - name: Test environment-independent protocol handshake run: node --test test/raw-protocol.test.mjs + + engine: + name: Engine (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.0.0 + with: + dotnet-version: | + 8.0.x + 10.0.100 + - run: dotnet test engine/OpenBfme.Engine.sln --nologo --configuration Release + + release-contracts: + name: Launcher and release contracts + runs-on: windows-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.0.0 + with: + dotnet-version: "10.0.100" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12.10" + - name: Install hash-pinned importer test dependencies + run: python -m pip install --require-hashes -r importer/requirements-win.txt + - name: Importer suite + env: + TEMP: ${{ runner.temp }} + TMP: ${{ runner.temp }} + TMPDIR: ${{ runner.temp }} + run: python -m pytest importer/tests -q + - name: Launcher security and rollback tests + run: dotnet run --project launcher/OpenBFME.Launcher.Tests/OpenBFME.Launcher.Tests.csproj --configuration Release + - name: Import reproducibility comparator tests + run: python -m unittest tools.release.test_compare_import_bundles -v + - name: Release firewall tests + shell: powershell + run: ./tools/release/Test-ReleaseTools.ps1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f35ebe6..9f51a90 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,27 +1,312 @@ -name: Release +name: windows release on: + workflow_dispatch: + inputs: + version: + description: "Release version without a leading v (manual validation does not publish)" + required: true + default: "0.0.0-local" + channel: + description: "Update channel" + required: true + type: choice + options: [playtest, stable, nightly] + default: playtest + run_acceptance: + description: "Run packaged BFME2 acceptance on the dedicated Windows runner" + required: true + type: boolean + default: false push: tags: ["v*"] permissions: - contents: write + contents: read jobs: - source-prerelease: - name: Publish source prerelease - runs-on: ubuntu-latest + build: + name: deterministic Windows packages + runs-on: windows-latest + permissions: + contents: read + id-token: write + attestations: write + outputs: + version: ${{ steps.identity.outputs.version }} + env: + GODOT_VERSION: "4.7" + GODOT_RELEASE: "4.7-stable" + GODOT_EDITOR_SHA512: "41645a908eb3181d6f2d1201ed7b6d6f095f6a23aaed8903d5d255277cc8d142814f3e6817f865b3cac142c39b8aff99280091d3bbdaa301517730b3ba0522b9" + GODOT_TEMPLATES_SHA512: "1035dfde4edcc2472bb0c0b9610ce3ee9302642c2b9957e9066372f9f6bb759ab250c8887551a66f0bc5f51bbd9a58bb45e33a0f29844e97615a9b1138c1120e" steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2 with: fetch-depth: 0 - - name: Require a version tag on main history - shell: bash + - uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.0.0 + with: + dotnet-version: | + 8.0.x + 10.0.100 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12.10" + + - name: Resolve immutable release identity + id: identity + shell: powershell run: | + $version = if ("${{ github.event_name }}" -eq "push") { + "${{ github.ref_name }}".Substring(1) + } else { + "${{ inputs.version }}" + } + if ($version -cnotmatch '^[0-9A-Za-z][0-9A-Za-z._-]{0,63}$') { throw "Unsafe version." } + if ("${{ github.event_name }}" -eq "push" -and + $version -cnotmatch '^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*)?$') { + throw "Release tags must use SemVer." + } + $channel = if ("${{ github.event_name }}" -eq "push") { + if ($version.Contains("-")) { "playtest" } else { "stable" } + } else { + "${{ inputs.channel }}" + } + "version=$version" >> $env:GITHUB_OUTPUT + "channel=$channel" >> $env:GITHUB_OUTPUT + + - name: Verify signed tag and protected branch ancestry + if: github.event_name == 'push' + env: + GH_TOKEN: ${{ github.token }} + shell: powershell + run: | + $ref = gh api "repos/$env:GITHUB_REPOSITORY/git/ref/tags/$env:GITHUB_REF_NAME" | ConvertFrom-Json + if ($ref.object.type -ne "tag") { throw "Release tag must be annotated and signed." } + $tag = gh api "repos/$env:GITHUB_REPOSITORY/git/tags/$($ref.object.sha)" | ConvertFrom-Json + if ($tag.verification.verified -ne $true) { + throw "GitHub did not verify the release tag signature: $($tag.verification.reason)" + } git fetch origin main --no-tags - git merge-base --is-ancestor "$GITHUB_SHA" "origin/main" - - name: Create GitHub prerelease + git merge-base --is-ancestor $env:GITHUB_SHA origin/main + if ($LASTEXITCODE -ne 0) { throw "Release commit is not an ancestor of origin/main." } + + - name: Run source gates + shell: powershell + run: | + python -m pip install --require-hashes -r importer/requirements-win.txt + python -m pytest importer/tests -q + dotnet test engine/OpenBfme.Engine.sln --nologo --configuration Release + dotnet run --project launcher/OpenBFME.Launcher.Tests/OpenBFME.Launcher.Tests.csproj --configuration Release + python -m unittest tools.release.test_compare_import_bundles -v + ./tools/test-export-scan.ps1 + ./tools/release/Test-ReleaseTools.ps1 + Push-Location tools/bfme-launcher-mcp + try { + npm.cmd ci + npm.cmd audit --audit-level=moderate + node --test test/raw-protocol.test.mjs + } finally { Pop-Location } + + - name: Download and verify Godot env: GH_TOKEN: ${{ github.token }} - shell: bash - run: gh release create "$GITHUB_REF_NAME" --verify-tag --generate-notes --prerelease --title "OpenBFME $GITHUB_REF_NAME" + shell: powershell + run: | + $toolRoot = Join-Path $env:RUNNER_TEMP "godot" + New-Item -ItemType Directory -Path $toolRoot | Out-Null + $editor = Join-Path $toolRoot "godot-editor.zip" + $templates = Join-Path $toolRoot "godot-templates.zip" + $releaseJson = gh api "repos/godotengine/godot/releases/tags/$env:GODOT_RELEASE" + if ($LASTEXITCODE -ne 0) { throw "Godot release lookup failed." } + $release = $releaseJson | ConvertFrom-Json + function Get-GodotReleaseAsset { + param([Parameter(Mandatory)][string]$Name, [Parameter(Mandatory)][string]$Destination) + $asset = @($release.assets | Where-Object name -CEQ $Name) + if ($asset.Count -ne 1 -or $asset[0].state -ne "uploaded") { + throw "Godot release asset identity is missing or ambiguous." + } + & curl.exe ` + --fail ` + --location ` + --silent ` + --show-error ` + --retry 5 ` + --retry-all-errors ` + --retry-delay 2 ` + --retry-max-time 120 ` + --header "Accept: application/octet-stream" ` + --header "Authorization: Bearer $env:GH_TOKEN" ` + --header "X-GitHub-Api-Version: 2022-11-28" ` + --output $Destination ` + "https://api.github.com/repos/godotengine/godot/releases/assets/$($asset[0].id)" + if ($LASTEXITCODE -ne 0) { throw "Godot release asset download failed." } + } + Get-GodotReleaseAsset ` + "Godot_v$($env:GODOT_RELEASE)_win64.exe.zip" ` + $editor + Get-GodotReleaseAsset ` + "Godot_v$($env:GODOT_RELEASE)_export_templates.tpz" ` + $templates + if ((Get-FileHash $editor -Algorithm SHA512).Hash.ToLowerInvariant() -ne $env:GODOT_EDITOR_SHA512) { throw "Godot editor hash mismatch." } + if ((Get-FileHash $templates -Algorithm SHA512).Hash.ToLowerInvariant() -ne $env:GODOT_TEMPLATES_SHA512) { throw "Godot templates hash mismatch." } + Expand-Archive $editor -DestinationPath (Join-Path $toolRoot "editor") + Expand-Archive $templates -DestinationPath (Join-Path $toolRoot "templates") + $templateTarget = Join-Path $env:APPDATA "Godot\export_templates\$env:GODOT_VERSION.stable" + New-Item -ItemType Directory -Path (Split-Path $templateTarget -Parent) -Force | Out-Null + Move-Item (Join-Path $toolRoot "templates\templates") $templateTarget + + - name: Export code-only game + shell: powershell + run: | + $stage = Join-Path $env:RUNNER_TEMP "openbfme-code-only" + $dist = Join-Path $env:RUNNER_TEMP "openbfme-release" + ./tools/release/Build-CodeOnlyExport.ps1 -RepositoryRoot $PWD -Destination $stage + $godot = Get-ChildItem (Join-Path $env:RUNNER_TEMP "godot\editor") -Filter "*console.exe" | Select-Object -First 1 -ExpandProperty FullName + ./tools/release/Invoke-GodotExport.ps1 -Godot $godot -Project (Join-Path $stage "game") -Output (Join-Path $dist "game\OpenBFME.exe") -LogRoot (Join-Path $dist "logs") + ./tools/release/Test-WindowsExport.ps1 -Executable (Join-Path $dist "game\OpenBFME.exe") -LogRoot (Join-Path $dist "logs") + + - name: Publish launcher and bundled importer source + shell: powershell + run: | + $dist = Join-Path $env:RUNNER_TEMP "openbfme-release" + $launcher = Join-Path $dist "launcher" + dotnet publish launcher/OpenBFME.Launcher/OpenBFME.Launcher.csproj --configuration Release --runtime win-x64 --self-contained true -p:PublishSingleFile=true -p:DebugType=None --output $launcher + $sourceArchive = Join-Path $env:RUNNER_TEMP "openbfme-bundled-source.zip" + git archive --format=zip --output $sourceArchive HEAD -- ` + importer/openbfme_importer ` + importer/blender ` + importer/profiles ` + importer/requirements-release-win.txt ` + contracts ` + tools/openbfme_import.py ` + tools/bootstrap-importer-python.ps1 + if ($LASTEXITCODE -ne 0) { throw "Failed to archive committed bundled source." } + Expand-Archive $sourceArchive -DestinationPath $launcher + $sourceIdentity = [ordered]@{ + schema = "openbfme.bundled-source-identity" + schemaVersion = 1 + commit = "${{ github.sha }}" + sourceClean = $true + } + [IO.File]::WriteAllText( + (Join-Path $launcher "release-identity.json"), + ($sourceIdentity | ConvertTo-Json) + "`n", + [Text.UTF8Encoding]::new($false) + ) + ./tools/release/New-PinnedPythonRuntime.ps1 ` + -SourcePython (Get-Command python).Source ` + -Destination (Join-Path $launcher "python") ` + -Requirements (Resolve-Path importer/requirements-release-win.txt) ` + -ImporterRoot (Resolve-Path importer) ` + -ImporterEntry (Join-Path $launcher "tools/openbfme_import.py") ` + -BundleRoot $launcher + ./tools/release/Test-LauncherHeadless.ps1 -Launcher (Join-Path $launcher "OpenBFME.Launcher.exe") + + - name: Package, scan, and attest + env: + OPENBFME_RELEASE_SIGNING_KEY: ${{ secrets.OPENBFME_RELEASE_SIGNING_KEY }} + shell: powershell + run: | + $version = "${{ steps.identity.outputs.version }}" + $dist = Join-Path $env:RUNNER_TEMP "openbfme-release" + $gameZip = Join-Path $dist "OpenBFME-$version-windows-x64.zip" + $launcherZip = Join-Path $dist "OpenBFME-Launcher-$version-windows-x64.zip" + Compress-Archive -Path (Join-Path $dist "game\*") -DestinationPath $gameZip -CompressionLevel Optimal + Compress-Archive -Path (Join-Path $dist "launcher\*") -DestinationPath $launcherZip -CompressionLevel Optimal + ./tools/release/Test-ReleaseArtifact.ps1 -Path $gameZip + ./tools/release/Test-ReleaseArtifact.ps1 -Path $launcherZip + ./tools/release/New-ReleaseManifest.ps1 -ReleaseRoot $dist -Version $version -Commit "${{ github.sha }}" -Channel "${{ steps.identity.outputs.channel }}" -Output (Join-Path $dist "release-manifest.json") + ./tools/release/Sign-ReleaseManifest.ps1 -Manifest (Join-Path $dist "release-manifest.json") -Output (Join-Path $dist "release-manifest.json.sig") + Get-ChildItem $dist -Filter "*.zip" | ForEach-Object { + "$(($_ | Get-FileHash -Algorithm SHA256).Hash.ToLowerInvariant()) $($_.Name)" + } | Set-Content (Join-Path $dist "SHA256SUMS.txt") -Encoding ascii + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: openbfme-${{ steps.identity.outputs.version }}-windows + path: | + ${{ runner.temp }}/openbfme-release/*.zip + ${{ runner.temp }}/openbfme-release/release-manifest.json + ${{ runner.temp }}/openbfme-release/release-manifest.json.sig + ${{ runner.temp }}/openbfme-release/SHA256SUMS.txt + if-no-files-found: error + retention-days: 14 + + - uses: actions/attest-build-provenance@43d14bc2b83dec42d39ecae14e916627a18bb661 # v3 + with: + subject-path: | + ${{ runner.temp }}/openbfme-release/*.zip + ${{ runner.temp }}/openbfme-release/release-manifest.json + ${{ runner.temp }}/openbfme-release/release-manifest.json.sig + + windows-vm-acceptance: + name: clean Windows BFME2 VM acceptance + if: github.event_name == 'push' || inputs.run_acceptance + needs: build + runs-on: [self-hosted, windows, x64, openbfme-release-vm] + timeout-minutes: 120 + permissions: + contents: read + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.2 + with: + fetch-depth: 1 + ref: ${{ github.sha }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12.10" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: openbfme-${{ needs.build.outputs.version }}-windows + path: release + - name: Run packaged launcher twice against BFME II + shell: powershell + run: | + if ([string]::IsNullOrWhiteSpace($env:BFME2_RETAIL_PATH)) { + throw "The Windows VM runner does not define BFME2_RETAIL_PATH." + } + ./tools/release/Invoke-WindowsVmAcceptance.ps1 ` + -ReleaseDirectory release ` + -RetailPath $env:BFME2_RETAIL_PATH ` + -RepositoryRoot $PWD ` + -ExpectedCommit "${{ github.sha }}" ` + -Receipt (Join-Path $env:RUNNER_TEMP "openbfme-vm-acceptance.json") + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: openbfme-${{ needs.build.outputs.version }}-vm-acceptance + path: ${{ runner.temp }}/openbfme-vm-acceptance.json + if-no-files-found: error + retention-days: 30 + + publish: + name: publish immutable GitHub release + if: github.event_name == 'push' + needs: [build, windows-vm-acceptance] + runs-on: windows-latest + environment: production-release + permissions: + contents: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: openbfme-${{ needs.build.outputs.version }}-windows + path: release + - name: Publish tag artifacts + env: + GH_TOKEN: ${{ github.token }} + shell: powershell + run: | + $version = "${{ needs.build.outputs.version }}" + $dist = Join-Path $PWD "release" + $args = @("release", "create", "${{ github.ref_name }}", + (Join-Path $dist "OpenBFME-$version-windows-x64.zip"), + (Join-Path $dist "OpenBFME-Launcher-$version-windows-x64.zip"), + (Join-Path $dist "release-manifest.json"), + (Join-Path $dist "release-manifest.json.sig"), + (Join-Path $dist "SHA256SUMS.txt"), + "--repo", "${{ github.repository }}", + "--verify-tag", + "--generate-notes", + "--title", "OpenBFME $version") + if ($version.Contains("-")) { $args += "--prerelease" } + gh @args diff --git a/DIRECTION.md b/DIRECTION.md index 01c740e..dd9faa8 100644 --- a/DIRECTION.md +++ b/DIRECTION.md @@ -1,45 +1,78 @@ -# Project direction +# OpenBFME product direction -OpenBFME aims to recreate BFME2 1.06 skirmish play in a modern, open-source, -moddable engine. +**Owner:** Jonathan, project owner +**Owns:** stable target, scope ladder, parity definition, and non-goals +**Does not own:** current hashes, gate results, task queue, or implementation detail +**Last verified commit:** `ad370cc9b02bdec600564cf1c606e70833faa97a` +**Update trigger:** the product target or scope changes +**Validation:** `contracts/bfme2-106-product-scope.json` -Players provide their own legally acquired copy of BFME2. OpenBFME converts the -required content locally and does not distribute original or converted game -assets. +## North star -## Scope +Create a modern, independently distributed and easily moddable RTS engine in +Godot that reproduces BFME2 1.06 skirmish play through measured +retail/original-game evidence. -| Goal | Status | -|---|---| -| Men versus Men on Fords of Isen II | In progress | -| Full Men faction on the selected maps | In progress | -| All six BFME2 factions and skirmish systems | In progress | -| Self-hosted multiplayer for up to eight players | Not started | -| Replays, observers, Create-a-Hero, and map tools | Not started | -| Accessibility and modern release tools | Not started | +The compatibility build uses locally converted content from a user-owned retail +installation. Retail and converted retail payloads stay under `.private`. A later +public distribution contains project-authored code and legal-safe fixtures only. -## Compatibility target +## Scope ladder -OpenBFME targets BFME2 version 1.06. Gameplay is compared with the original game -where possible; having a file or model available does not mean the feature is -finished. +1. Complete and freeze Men versus Men on Fords of Isen II. +2. Complete the full Men faction, including every BFME2 1.06 Men hero, on the + selected five-map oracle set. +3. Complete all six BFME2 factions, Ring mechanics, naval gameplay, neutral + objects, and official skirmish/multiplayer maps. +4. Ship self-hosted local, listen, and dedicated-server play for up to eight + players using deterministic lockstep. +5. Complete Create-a-Hero, the skirmish shell, saves, replays, observers, and + custom-map/scenario tooling. +6. Add modern accessibility, HD presentation packs, mod management, safe mode, + diagnostics, and rollback updates without changing the parity profile. -## Not in scope +RotWK is a separate future overlay and cannot change BFME2 1.06 evidence. -- The Good and Evil campaigns -- Campaign maps and scripting -- War of the Ring -- Rise of the Witch-king support during the BFME2 phase -- Ranked services or a mandatory online account +The ladder orders acceptance, not code presence. The development tree already +carries unaccepted surfaces from steps 2 and 3 — six faction runtimes, a +five-map development set, heroes, powers, and spellbooks — while step 1 remains +the only active acceptance target. Presence of those surfaces does not advance +the ladder; only the evidence gates in [docs/MILESTONE_CURRENT.md](docs/MILESTONE_CURRENT.md) +do. Current runtime evidence lives in [STATUS.md](STATUS.md). -## Long-term technical direction +## Meaning of parity -- Godot for presentation, input, interface, audio, and desktop integration -- A deterministic simulation suitable for replays and multiplayer -- Self-hosted local, listen-server, and dedicated-server play -- Up to eight players -- Separate versioning for gameplay mods and presentation mods -- No retail or converted retail assets in the repository or public downloads +"Near 1:1" means every included capability is discovered from the effective +BFME2 1.06 source corpus and has the required source, conversion, runtime, +simulation, presentation, oracle, and reliability evidence. -See [PLAN.md](PLAN.md) for the development order and [STATUS.md](STATUS.md) for -current progress. +INI presence and converted-asset counts are not parity. Unknown, ambiguous, +unsupported, substituted, or unclassified requirements fail closed. + +## Permanent product constraints + +- Eight players maximum. +- Godot owns presentation, input, UI, audio, and desktop integration. +- Pure C# owns deterministic authoritative simulation. +- Production simulation targets 30 Hz; presentation remains render-rate independent. +- Multiplayer is server-refereed deterministic lockstep and self-hostable. +- No Steam, ranked-service, or mandatory-account dependency. +- Gameplay and presentation mods are versioned and hashed separately. +- Private parity never silently uses synthetic or generic replacement art. +- The Good and Evil campaigns, campaign maps and scripting, and War of the Ring + are outside the OpenBFME product scope. They are not later roadmap promises. + +## Active milestone + +The binary active contract is [docs/MILESTONE_CURRENT.md](docs/MILESTONE_CURRENT.md). +Current evidence and blockers live only in [STATUS.md](STATUS.md). + +## Non-goals before M2 acceptance + +- New synthetic proof-stage features. +- Multiplayer or RotWK implementation. +- Campaign or War of the Ring implementation at any milestone. +- Broad importer, presentation, or architecture refactors. +- Publishing a public release before the code-only export, signed update, + deterministic importer, clean Windows VM, and containment gates pass. +- Declaring completion without the identity-bound oracle and reliability gate. diff --git a/README.md b/README.md index 40ec7fe..eacd282 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,6 @@ > requires a lawfully acquired BFME2 1.06 installation and converts content > locally on your computer. -> [!NOTE] -> OpenBFME is an early development project. Expect unfinished features, bugs, -> and breaking changes. - ## What is OpenBFME? OpenBFME is rebuilding the skirmish side of *The Battle for Middle-earth II* in @@ -42,26 +38,47 @@ The importer understands BFME2's source formats; the game runtime loads a versioned pack generated privately on the user's machine. Proprietary retail content stays outside Git and outside public releases. -## Current state - -OpenBFME can import BFME2 data and run an early skirmish experience in Godot. -Men versus Men on Fords of Isen II is the most complete part of the project. -Other factions and maps are under active development and are not yet ready for -normal play. - -| Feature | Status | -|---|---| -| BFME2 1.06 importer | Completed | -| Local private content packs | Completed | -| Main menu and skirmish setup | In progress | -| Men versus Men on Fords of Isen II | In progress | -| All six BFME2 factions | In progress | -| Five-map development set | In progress | -| Multiplayer and dedicated servers | Not started | -| Public installer | Not started | - -Campaigns, War of the Ring, and Rise of the Witch-king are not in the current -project scope. See [STATUS.md](STATUS.md) for known problems and test results. +## Where the development tree is today + +The current target is a polished Men-versus-Men skirmish on Fords of Isen II. +The codebase contains broader systems and experimental faction paths, but they +are not all at the same quality level. This table uses only three statuses: + +- **Completed** means the implementation passes its applicable current tests. +- **In progress** means meaningful code and tests exist, but parity or release + acceptance is incomplete. +- **Not started** means there is no supported implementation. + +| Feature | BFME II | Rise of the Witch-king | OpenBFME status | +|---|---|---|---| +| Local retail discovery and fail-closed archive identity | Required source | Future overlay | Completed for BFME II 1.06 | +| Core skirmish loop | Included | Inherited and expanded | In progress | +| Men-versus-Men on Fords of Isen II | Included | Inherited | In progress; primary playable slice | +| Six BFME II factions | Included | Additional units and balance changes | In progress; coverage is uneven | +| Angmar | Not included | New faction | Not started; outside the current release target | +| Building, production, combat, upgrades, powers, and heroes | Included | Expanded | In progress | +| Skirmish AI | Included | Expanded | In progress | +| LAN/online multiplayer | Included | Inherited | In progress | +| Custom fortresses and walls | Included | Inherited | In progress | +| Create-a-Hero | Included | New Troll class, weapons, and armor | Not started | +| Campaigns | Good and Evil campaigns | Angmar campaign | Not started; outside project scope | +| War of the Ring | Included | Expanded persistence and siege rules | Not started; outside project scope | +| Code-only Windows export | Not applicable | Not applicable | Completed | +| Launcher, signed update manifest, updates, and rollback | Not applicable | Not applicable | Completed | +| Repeatable packaged BFME II import | Required source | Not the current release target | Completed | +| Clean Windows release VM and public release | Not applicable | Not applicable | In progress | + +EA's original announcements are the reference for the high-level comparison: +[BFME II introduced custom heroes, fortresses, walls, and War of the Ring](https://ir.ea.com/press-releases/press-release-details/2006/EA-Ships-The-Lord-of-the-Rings-The-Battle-for-Middle-earth-II-and-The-Lord-of-the-Rings-The-Battle-for-Middle-earth-II-Collectors-Edition-Highly-Anticipated-PC-Game-Ships-Nationwide-Today/default.aspx); +[Rise of the Witch-king added Angmar, faction units, a campaign, expanded +Create-a-Hero, and an upgraded War of the Ring](https://ir.ea.com/press-releases/press-release-details/2006/EAs-The-Lord-of-the-Rings-The-Battle-for-Middle-earth-II-The-Rise-of-the-Witch-king-Has-Shipped-for-the-PC/default.aspx). +Project status comes from current code and gates, not those marketing pages. +See [STATUS.md](STATUS.md) for current evidence and blockers. + +The packaged launcher has produced two byte-identical BFME II Men/Fords packs +from separate empty local states, and the resulting Windows export starts with +that selected pack. Public release publication remains blocked until the same +test passes on a dedicated clean Windows VM. ## Why this project exists @@ -109,6 +126,22 @@ The current workflow is Windows-first and intended for developers. You need a lawfully acquired BFME2 1.06 installation, Godot 4.7, Python 3.12, and the .NET SDK selected by `global.json`. +The guided onboarding wizard checks prerequisites, validates your install +fail-closed, converts or verifies the Men content pack, and runs the headless +verification gates: + +```bat +python tools\onboard.py +``` + +Non-interactive equivalent (CI or scripted setup): + +```bat +python tools\onboard.py --install "D:\Games\BFME2" --godot "C:\Tools\Godot\Godot_v4.7-stable_win64_console.exe" --yes +``` + +The manual command path still works: + ```bat set OPENBFME_GODOT=C:\Tools\Godot\Godot_v4.7-stable_win64.exe run_doctor.bat @@ -116,29 +149,28 @@ run_importer.bat "D:\Games\BFME2" run_retail_slice.bat ``` -Use your actual Godot and BFME2 paths. Read the full -[getting-started guide](docs/GETTING_STARTED.md) before importing. +Use your actual Godot and BFME2 paths. Read the +[onboarding guide](docs/ONBOARDING.md) for the ten-minute walkthrough and the +[getting-started guide](docs/GETTING_STARTED.md) for the full background before +importing. ## Roadmap -| Goal | Status | -|---|---| -| Import BFME2 1.06 content locally | Completed | -| Finish Men versus Men on Fords of Isen II | In progress | -| Finish the Men faction across the selected maps | In progress | -| Finish all six BFME2 factions and skirmish systems | In progress | -| Add self-hosted multiplayer for up to eight players | Not started | -| Add replays, observers, Create-a-Hero, and broader modding tools | Not started | -| Package a polished public installer | Not started | +1. Finish the Men-versus-Men Fords of Isen II release slice. +2. Ship the code-only Windows launcher, repeatable local importer, updates, and + rollback. +3. Expand BFME II skirmish coverage to more Men units, maps, and factions. +4. Harden multiplayer and modding after the local skirmish release is stable. -Campaign material and War of the Ring are not planned. More detail is available -in [DIRECTION.md](DIRECTION.md). +Campaign material and War of the Ring are not part of this roadmap. The stable +scope and non-goals live in [DIRECTION.md](DIRECTION.md). ## Find your way around | If you want to... | Start here | |---|---| | Understand the project in five minutes | [Documentation hub](docs/README.md) | +| Set up a fresh machine in ten minutes | [Onboarding](docs/ONBOARDING.md) | | Install and run the developer build | [Getting started](docs/GETTING_STARTED.md) | | Check current passes and failures | [Status](STATUS.md) | | Understand the engine boundaries | [Architecture](docs/ARCHITECTURE.md) | @@ -149,11 +181,17 @@ in [DIRECTION.md](DIRECTION.md). | Understand the use of AI | [AI development](docs/AI_DEVELOPMENT.md) | | Ask a common question | [FAQ](docs/FAQ.md) | -## AI-assisted development +## Built with AI, judged by evidence OpenBFME has been built with extensive AI assistance under human direction and -testing. Fable 5, ChatGPT Sol, and Kimi K3 have all contributed to the project. -AI-generated work is reviewed and tested like any other contribution. See +testing. The project owner reports that Fable 5, ChatGPT Sol, and Kimi K3 +contributed substantial implementation and review work. The current Git history +does not preserve model-level attribution for individual changes, so those +credits are owner testimony rather than repository-verifiable authorship. + +That origin is part of the experiment, not proof that the result is correct. +Claims are accepted only when backed by source evidence, focused tests, runtime +behavior, original-game comparison, and human review. See [docs/AI_DEVELOPMENT.md](docs/AI_DEVELOPMENT.md). ## Contributing @@ -168,8 +206,8 @@ with [CONTRIBUTING.md](CONTRIBUTING.md). ## License and legal notice -OpenBFME source is distributed under the GNU General Public License v3.0. The -repository carries its own [LICENSE](LICENSE). That license applies +The proposed public source is distributed under the GNU General Public License +v3.0; the repository now carries its own [LICENSE](LICENSE). That license applies to code the project is authorized to license, not to *The Lord of the Rings*, BFME2, or third-party content. Third-party provenance and notice review remains a publication gate. diff --git a/docs/LAUNCHER_AND_RELEASES.md b/docs/LAUNCHER_AND_RELEASES.md new file mode 100644 index 0000000..4041edc --- /dev/null +++ b/docs/LAUNCHER_AND_RELEASES.md @@ -0,0 +1,124 @@ +# Launcher and Windows releases + +OpenBFME releases are code-only. They contain the engine, launcher, importer, +and pinned conversion runtime, but never a retail game pack. A player supplies +a locally installed copy of BFME II 1.06. RotWK is a separate future overlay +and is not part of the current release target. + +## Player workflow + +1. Download `OpenBFME-Launcher--windows-x64.zip` from a GitHub Release. +2. Extract it to a new folder and run `OpenBFME.Launcher.exe`. +3. Select the local BFME II installation and choose **Import BFME II (Men)**. +4. Wait for tool verification, conversion, pack audit, and local selection to + complete. Retail files and converted output stay on that PC. +5. Choose **Play OpenBFME**. + +The first import downloads hash-pinned Blender, OpenSAGE, and FFmpeg archives. +The launcher includes its own pinned Python runtime. Later imports reuse the +attested tools and cache. + +The stable channel checks +`https://github.com/Ancalgonn/open-bfme-engine/releases/latest/download/release-manifest.json`. +The launcher verifies the manifest's detached RSA signature before reading it. +The signed manifest binds the repository, version, channel, full commit, +package name, compressed and expanded byte sizes, SHA-256, and approved HTTPS +download URL. The updater downloads into a new immutable version directory, +verifies every installed file before selection, and retains only the current +and previous verified versions. + +## Launcher flags + +```text +--channel stable|playtest|nightly +--manifest-url +--install-root +--no-update +--verify-only +--headless +--import-bfme2 --bfme2-path +``` + +Stable has a default update feed. Playtest and nightly builds require an +explicit immutable manifest URL until those channels have dedicated feeds. + +## Maintainer release flow + +The `windows release` GitHub workflow has two entry points: + +- `workflow_dispatch` builds and uploads a temporary Actions artifact without + publishing a GitHub Release. +- A tag matching `v*` builds, verifies, uploads, and publishes an immutable + GitHub Release. + +The build job: + +1. runs engine, launcher, reproducibility-comparator, export-firewall, and raw + launcher-protocol tests; +2. downloads Godot 4.7 and its templates and verifies fixed SHA-512 digests; +3. stages `game/` outside the checkout while excluding `.private`, generated + caches, captures, and `game/data/base`; +4. imports and exports with Godot and fails on logged warnings or errors even + when Godot exits zero; +5. launches the produced executable headlessly as a smoke test; +6. publishes a self-contained Windows launcher with the importer and Python + runtime; +7. scans both ZIPs for unsafe paths, retail formats, private paths, game packs, + and agent instructions; +8. creates and signs `release-manifest.json`, writes `SHA256SUMS.txt`, and + records GitHub build provenance; and +9. grants GitHub write permission only to the separate tag publication job. + +For a release tag, a second job must run on an isolated runner labeled +`self-hosted`, `windows`, `x64`, and `openbfme-release-vm`. It verifies the +manifest signature and both ZIPs, runs the packaged launcher twice against a +lawfully installed BFME II copy from empty state, compares the complete packs +byte-for-byte, and smoke-launches the packaged game with the selected pack. +The publication job cannot start unless this VM gate passes. + +| Release component | Status | +|---|---| +| Code-only Windows export and smoke test | Completed | +| Launcher updates, self-update, integrity checks, and rollback | Completed | +| Signed manifests and build provenance | Completed | +| Two independent local BFME II imports with identical packs | Completed | +| Dedicated clean Windows acceptance runner | In progress | +| Authenticode signing for the Windows executables | Not started | +| First public tagged release | Not started | + +Create a stable release: + +```powershell +git tag -s v0.1.0 -m "OpenBFME 0.1.0" +git push origin v0.1.0 +``` + +Use a suffix such as `v0.1.0-playtest.1` for a prerelease. Do not tag a dirty +tree, a commit that lacks the required milestone evidence, or a revision whose +code-only export manifest has not been reviewed. + +## Local release validation + +```powershell +powershell -ExecutionPolicy Bypass -File tools/release/Test-ReleaseTools.ps1 +dotnet run --project launcher/OpenBFME.Launcher.Tests/OpenBFME.Launcher.Tests.csproj -c Release +python -m unittest tools.release.test_compare_import_bundles -v +``` + +The GitHub workflow is the source of truth for packaging. Local runs are a +preflight and do not authorize publication. + +The manifest signature protects the update feed. Authenticode is a separate +Windows publisher-identity requirement and needs a trusted external +code-signing certificate or service; the release should remain a playtest until +that is configured. + +## Failure and recovery + +- A package hash, size, URL, or archive-path mismatch stops installation before + the current version changes. +- A failed import does not select an incomplete pack. +- **Roll back** swaps to the previous verified engine version. +- Diagnostics shown by the launcher redact absolute retail paths. + +Campaigns and War of the Ring are not supported by this release system. diff --git a/docs/README.md b/docs/README.md index da53f2d..1c1f672 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,11 +7,15 @@ entire archive. ## Start here 1. [Project README](../README.md) — purpose, present state, and roadmap. -2. [Getting started](GETTING_STARTED.md) — prepare a BFME2 1.06 installation, - run the doctor, import content locally, and launch the development slice. -3. [FAQ](FAQ.md) — legality, assets, AI, supported scope, platforms, and common +2. [Onboarding](ONBOARDING.md) — the ten-minute guided path: run + `tools/onboard.py` to check prerequisites, validate your install, convert + or verify content, and run the verification gates. +3. [Getting started](GETTING_STARTED.md) — the manual workflow: prepare a + BFME2 1.06 installation, run the doctor, import content locally, and launch + the development slice. +4. [FAQ](FAQ.md) — legality, assets, AI, supported scope, platforms, and common misconceptions. -4. [Contributing](../CONTRIBUTING.md) — contribution boundaries, tests, and the +5. [Contributing](../CONTRIBUTING.md) — contribution boundaries, tests, and the retail-content firewall. ## Project truth @@ -39,6 +43,8 @@ documents should link to it rather than copying volatile claims. | Deterministic simulation and networking target | [SIMULATION_PROTOCOL.md](SIMULATION_PROTOCOL.md) | | Mod and content-pack direction | [MODDING.md](MODDING.md) | | Code-only public distribution | [RELEASE_POLICY.md](RELEASE_POLICY.md) | +| Launcher and Windows release workflow | [LAUNCHER_AND_RELEASES.md](LAUNCHER_AND_RELEASES.md) | +| Clean Windows VM acceptance | [WINDOWS_VM_ACCEPTANCE.md](WINDOWS_VM_ACCEPTANCE.md) | | Third-party tools and provenance | [THIRD_PARTY.md](THIRD_PARTY.md) | | AI-assisted development | [AI_DEVELOPMENT.md](AI_DEVELOPMENT.md) | diff --git a/docs/WINDOWS_VM_ACCEPTANCE.md b/docs/WINDOWS_VM_ACCEPTANCE.md new file mode 100644 index 0000000..3b03802 --- /dev/null +++ b/docs/WINDOWS_VM_ACCEPTANCE.md @@ -0,0 +1,82 @@ +# Windows VM release acceptance + +This is the end-to-end release gate for a clean Windows guest. It must use a +disposable VM or a checkpoint that can be restored. It must not reuse a +developer content pack or the paused reverse-BFME worker fleet. + +## Required guest + +- Windows 11 or Windows Server 2022 x64 +- at least 4 CPU cores, 16 GB RAM, and 80 GB free disk +- working QEMU guest agent or another bounded remote-execution channel +- network access to GitHub, Blender, and the pinned FFmpeg source +- a lawfully installed BFME II 1.06 copy +- RotWK 2.01 only when the optional Angmar test is requested + +The GitHub runner must have the labels `self-hosted`, `windows`, `x64`, and +`openbfme-release-vm`. Configure `BFME2_RETAIL_PATH` as a machine-level +environment variable on the guest; do not store the retail path or any retail +payload in GitHub variables, secrets, caches, or artifacts. The runner should +accept only release-tag jobs after the build job has verified the signed tag +and its ancestry on `main`. + +Record the VM identity, base snapshot, Windows build, release commit, release +manifest SHA-256, and retail installation identity before testing. Never copy +retail data into CI artifacts or logs. + +## Acceptance sequence + +1. Restore the clean checkpoint. +2. Download the release ZIPs and `release-manifest.json` from the tested GitHub + Actions run or release. +3. Verify `SHA256SUMS.txt`. +4. Extract and start the launcher. +5. Import BFME II Men into empty local launcher state. +6. Verify that the selected pack is valid and that no absolute retail path is + present in its public receipt. +7. Start OpenBFME, reach the menu, open skirmish setup, and begin the supported + Men-versus-Men Fords of Isen II slice. +8. Close the game, install the next synthetic engine version, verify selection, + then roll back and verify the prior commit identity. +9. Restore the checkpoint and repeat the BFME II import. +10. Compare the two complete pack trees with + `tools/release/compare_import_bundles.py`. + +The checked-in `tools/release/Invoke-WindowsVmAcceptance.ps1` performs the +package, signature, double-import, full-tree comparison, and exported-game +smoke checks. Snapshot restoration and ephemeral runner registration remain +infrastructure responsibilities so that every tag starts from the approved +clean guest image. + +The comparison is whole-tree and byte-exact. It inventories every regular file, +streams SHA-256, rejects links and case-colliding Windows paths, checks required +asset-family counts, and writes a payload-free receipt. A different texture, +model, animation, skeleton, audio file, map, rule, path, size, or byte fails the +gate. + +Example: + +```powershell +python tools/release/compare_import_bundles.py ` + C:\OpenBFME-A\content-packs\ ` + C:\OpenBFME-B\content-packs\ ` + --game bfme2 ` + --profile men-fords-v0 ` + --release-commit <40-character-commit> ` + --require-family textures=1 ` + --require-family models=1 ` + --require-family animations=1 ` + --receipt C:\OpenBFME-Proof\bfme2-men-repro.json +``` + +Do not record the retail path, asset names, payload bytes, screenshots containing +private material, or private pack manifests in a public artifact. Preserve only +the sanitized receipt, launcher logs, process exit codes, version identities, +and pass/fail summary. + +## Pass condition + +The VM gate passes only when the packaged launcher starts on the clean guest, +the importer builds and selects a pack without manual developer dependencies, +the game launches with that pack, update and rollback preserve the correct +commit identities, and the two clean imports have the same canonical digest. diff --git a/docs/assets/openbfme-readme-banner.png b/docs/assets/openbfme-readme-banner.png index 1774c6c..65ceae2 100644 Binary files a/docs/assets/openbfme-readme-banner.png and b/docs/assets/openbfme-readme-banner.png differ diff --git a/engine/OpenBfme.Engine.sln b/engine/OpenBfme.Engine.sln new file mode 100644 index 0000000..87f0700 --- /dev/null +++ b/engine/OpenBfme.Engine.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenBfme.Sim", "OpenBfme.Sim\OpenBfme.Sim.csproj", "{9C415658-75B3-4422-BC51-3A4BE0F54B88}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OpenBfme.Sim.Tests", "OpenBfme.Sim.Tests\OpenBfme.Sim.Tests.csproj", "{A9684FB2-CDFD-4C44-821A-864984A0BE0C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Debug|x64.ActiveCfg = Debug|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Debug|x64.Build.0 = Debug|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Debug|x86.ActiveCfg = Debug|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Debug|x86.Build.0 = Debug|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Release|Any CPU.Build.0 = Release|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Release|x64.ActiveCfg = Release|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Release|x64.Build.0 = Release|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Release|x86.ActiveCfg = Release|Any CPU + {9C415658-75B3-4422-BC51-3A4BE0F54B88}.Release|x86.Build.0 = Release|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Debug|x64.ActiveCfg = Debug|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Debug|x64.Build.0 = Debug|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Debug|x86.ActiveCfg = Debug|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Debug|x86.Build.0 = Debug|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Release|Any CPU.Build.0 = Release|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Release|x64.ActiveCfg = Release|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Release|x64.Build.0 = Release|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Release|x86.ActiveCfg = Release|Any CPU + {A9684FB2-CDFD-4C44-821A-864984A0BE0C}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/engine/OpenBfme.Sim.Tests/BurnDownModuleTests.cs b/engine/OpenBfme.Sim.Tests/BurnDownModuleTests.cs new file mode 100644 index 0000000..7da1abc --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/BurnDownModuleTests.cs @@ -0,0 +1,405 @@ +using OpenBfme.Sim; +using Xunit; + +namespace OpenBfme.Sim.Tests; + +/// +/// Burn-down batch tests: production economy (cost debit/reject/refund), retail +/// spawn phase (command_tick + build_ticks, exact), exit->rally walk, and the +/// three new gap modules — HordeContain (member-slot health delegation), +/// BezierProjectile-lite (straight-line flight, typed damage on arrival), and +/// AttributeModifierAura-lite (radius armor aura, additive stacking clamped to +/// 10000 bp, table rebuilt at end of tick). +/// +public class BurnDownModuleTests +{ + private const long SoldierCost = 300; + private const long SoldierBuildTicks = 20; + + private static SimConfig Config(ulong seed = 11) => new( + new[] + { + new ObjectTemplate("keep", new[] + { + new ModuleSpec(StructureBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 900 }), + new ModuleSpec(ProductionModule.TypeName, new Dictionary + { + ["Build:soldier"] = SoldierBuildTicks, + ["Cost:soldier"] = SoldierCost, + ["RallyXRaw"] = Fixed64.FromInt(4).Raw, + ["RallyYRaw"] = Fixed64.FromInt(3).Raw, + }), + }), + new ObjectTemplate("soldier", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 120 }), + new ModuleSpec(LinearMoverModule.TypeName, new Dictionary + { + ["SpeedPerTickRaw"] = Fixed64.FromFraction(1, 2).Raw, + }), + }), + new ObjectTemplate("horde", new[] + { + new ModuleSpec(HordeContainModule.TypeName, new Dictionary + { + ["MemberCount"] = 3, + ["MemberHealth"] = 100, + }), + }), + new ObjectTemplate("arrow", new[] + { + new ModuleSpec(BezierProjectileModule.TypeName, new Dictionary + { + ["FlightTicks"] = 8, + ["Damage"] = 40, + }, new Dictionary + { + ["DamageType"] = DamageTypes.Siege, + }), + }), + new ObjectTemplate("target", new[] + { + new ModuleSpec(ArmorModule.TypeName, new Dictionary { ["Armor:siege"] = 5_000 }), + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 200 }), + }), + new ObjectTemplate("grunt", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 200 }), + new ModuleSpec(LinearMoverModule.TypeName, new Dictionary + { + ["SpeedPerTickRaw"] = Fixed64.FromFraction(1, 2).Raw, + }), + }), + new ObjectTemplate("banner", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 100 }), + new ModuleSpec(AttributeModifierAuraModule.TypeName, new Dictionary + { + ["RadiusRaw"] = Fixed64.FromInt(5).Raw, + ["RecomputeTicks"] = 4, + ["ArmorBonusBp"] = 2_500, + }), + }), + }, + seed, + teamCount: 2); + + private static SimWorld NewWorld(ulong seed = 11) => new(Config(seed), ModuleRegistry.CreateDefault()); + + private static FixedVector2 At(int x, int y) => new(Fixed64.FromInt(x), Fixed64.FromInt(y)); + + // ---- production economy ------------------------------------------------- + + [Fact] + public void ProductionCostDebitsRejectsUnaffordableAndRefundsOnCancel() + { + var world = NewWorld(); + world.AddTeamResources(0, 500); + var keep = world.SpawnObject("keep", 0, FixedVector2.Zero); + var production = keep.FindModule()!; + + Assert.True(production.TryQueue(world, keep, "soldier")); + Assert.Equal(500 - SoldierCost, world.TeamResources(0)); + + // 200 < 300: unaffordable requests are rejected WITHOUT queueing. + Assert.False(production.TryQueue(world, keep, "soldier")); + Assert.Equal(1, production.QueueLength); + Assert.Equal(500 - SoldierCost, world.TeamResources(0)); + + // Cancellation refunds the full cost. + Assert.True(production.TryCancel(world, keep, 0)); + Assert.Equal(0, production.QueueLength); + Assert.Equal(500, world.TeamResources(0)); + Assert.False(production.TryCancel(world, keep, 0)); + Assert.Equal(500, world.TeamResources(0)); + } + + [Fact] + public void CancelProductionCommandRefundsThroughTeamValidation() + { + var world = NewWorld(); + world.AddTeamResources(0, 1000); + var keep = world.SpawnObject("keep", 0, FixedVector2.Zero); + world.SubmitCommand(TestWorlds.Command(1, 0, 0, "queue_production", + ("id", CommandValue.OfLong(keep.Id)), ("template", CommandValue.OfString("soldier")))); + // Wrong team may not cancel (no refund, entry stays). + world.SubmitCommand(TestWorlds.Command(2, 1, 0, "cancel_production", + ("id", CommandValue.OfLong(keep.Id)), ("index", CommandValue.OfLong(0)))); + world.Advance(2); + Assert.Equal(1, keep.FindModule()!.QueueLength); + Assert.Equal(1000 - SoldierCost, world.TeamResources(0)); + // Right team cancels and is refunded. + world.SubmitCommand(TestWorlds.Command(3, 0, 1, "cancel_production", + ("id", CommandValue.OfLong(keep.Id)), ("index", CommandValue.OfLong(0)))); + world.Advance(1); + Assert.Equal(0, keep.FindModule()!.QueueLength); + Assert.Equal(1000, world.TeamResources(0)); + } + + [Fact] + public void SpawnLandsExactlyAtCommandTickPlusBuildTicks() + { + var world = NewWorld(); + world.AddTeamResources(0, 500); + var keep = world.SpawnObject("keep", 0, FixedVector2.Zero); + const int commandTick = 5; + world.SubmitCommand(TestWorlds.Command(commandTick, 0, 0, "queue_production", + ("id", CommandValue.OfLong(keep.Id)), ("template", CommandValue.OfString("soldier")))); + + // Cost is debited when the command applies, at tick start of commandTick. + world.Advance(commandTick); + Assert.Equal(500 - SoldierCost, world.TeamResources(0)); + + // Retail spawn phase: the soldier exists after tick commandTick + build_ticks, + // and not one tick sooner. + world.Advance((int)SoldierBuildTicks - 1); // now at commandTick + build_ticks - 1 + Assert.Single(world.Objects); + world.Advance(1); // commandTick + build_ticks + Assert.Equal(2, world.Objects.Count); + } + + [Fact] + public void SpawnedUnitWalksFromExitToRallyPoint() + { + var world = NewWorld(); + world.AddTeamResources(0, 500); + var keep = world.SpawnObject("keep", 0, At(10, 10)); + Assert.True(keep.FindModule()!.TryQueue(world, keep, "soldier")); + + // Queued pre-tick: build ticks 1..20, spawn during tick 21 at the exit + // point (default ExitOffset +2,0), rally walk begins next tick. + world.Advance(21); + var soldier = world.Objects.Values.Single(o => o.TemplateName == "soldier"); + Assert.Equal(At(12, 10), soldier.Position); + + // RallyX/YRaw are producer-relative: (10,10) + (4,3) = (14,13). The + // LinearMover snaps exactly onto the rally point. + world.Advance(40); + Assert.Equal(At(14, 13), soldier.Position); + } + + // ---- HordeContain -------------------------------------------------------- + + [Fact] + public void HordeDamageFillsMemberSlotsDeterministicallyAndKillsThrough() + { + var world = NewWorld(); + var horde = world.SpawnObject("horde", 1, FixedVector2.Zero); + var contain = horde.FindModule()!; + Assert.Equal(3, contain.MemberCount); + Assert.Equal(300, contain.TotalHealth); + + // 150 damage: slot 0 dies (100), overflow 50 into slot 1, slot 2 untouched. + world.DealDamage(horde, 150); + Assert.Equal(0, contain.MemberHealthAt(0)); + Assert.Equal(50, contain.MemberHealthAt(1)); + Assert.Equal(100, contain.MemberHealthAt(2)); + Assert.Equal(2, contain.AliveMemberCount); + + // Down to the last hit point of the last member. + world.DealDamage(horde, 149); + Assert.Equal(1, contain.AliveMemberCount); + Assert.Equal(1, contain.TotalHealth); + Assert.Single(world.Objects); + + // Final point: the horde dies through the normal death pipeline. + world.DealDamage(horde, 1); + world.Advance(1); + Assert.Empty(world.Objects); + } + + // ---- BezierProjectile-lite ----------------------------------------------- + + [Fact] + public void ProjectileFliesForFlightTicksThenDealsTypedDamageAndExpires() + { + var world = NewWorld(); + var target = world.SpawnObject("target", 1, At(8, 0)); + var arrow = world.SpawnObject("arrow", 0, FixedVector2.Zero); + arrow.FindModule()!.Launch(world, arrow, target.Id); + + // Flight ticks 1..7: no damage yet, arrow closing on the target. + world.Advance(7); + Assert.Equal(200, target.FindModule()!.Health); + Assert.Equal(2, world.Objects.Count); + + // Arrival on the 8th update: 40 siege into Armor:siege 5000bp = 20; the + // spent projectile leaves the world the same tick. + world.Advance(1); + Assert.Equal(180, target.FindModule()!.Health); + Assert.Single(world.Objects); + } + + [Fact] + public void ProjectileExpiresHarmlesslyWhenTargetDiesMidFlight() + { + var world = NewWorld(); + var target = world.SpawnObject("target", 1, At(8, 0)); + var arrow = world.SpawnObject("arrow", 0, FixedVector2.Zero); + arrow.FindModule()!.Launch(world, arrow, target.Id); + world.Advance(3); + world.DealDamage(target, 10_000); // dies; removed next tick + world.Advance(5); + Assert.Empty(world.Objects); // no crash, arrow expired without damage + } + + // ---- AttributeModifierAura-lite ------------------------------------------- + + [Fact] + public void AuraReducesDamageForAlliesInRangeOnlyAndStacksAdditively() + { + var world = NewWorld(); + var banner = world.SpawnObject("banner", 0, FixedVector2.Zero); + var allyNear = world.SpawnObject("grunt", 0, At(3, 0)); + var allyFar = world.SpawnObject("grunt", 0, At(30, 0)); + var enemyNear = world.SpawnObject("grunt", 1, At(2, 0)); + world.Advance(1); // aura scans on its first update; table rebuilt at end of tick + + Assert.Equal(2_500, world.AuraArmorBonusBp(allyNear.Id)); + Assert.Equal(0, world.AuraArmorBonusBp(allyFar.Id)); + Assert.Equal(0, world.AuraArmorBonusBp(enemyNear.Id)); + Assert.Equal(0, world.AuraArmorBonusBp(banner.Id)); // carriers do not buff themselves + + world.DealDamage(allyNear, 100); + world.DealDamage(allyFar, 100); + world.DealDamage(enemyNear, 100); + Assert.Equal(200 - 75, allyNear.FindModule()!.Health); + Assert.Equal(200 - 100, allyFar.FindModule()!.Health); + Assert.Equal(200 - 100, enemyNear.FindModule()!.Health); + + // STACKING RULE: contributions from multiple carriers ADD (clamped to + // 10000 bp at application). Two 2500bp banners = 5000bp = half damage. + world.SpawnObject("banner", 0, At(1, 0)); + world.Advance(1); + Assert.Equal(5_000, world.AuraArmorBonusBp(allyNear.Id)); + world.DealDamage(allyNear, 100); + Assert.Equal(200 - 75 - 50, allyNear.FindModule()!.Health); + } + + [Fact] + public void AuraMembershipFollowsMovementOnTheRecomputeCadence() + { + var world = NewWorld(); + world.SpawnObject("banner", 0, FixedVector2.Zero); + var grunt = world.SpawnObject("grunt", 0, At(3, 0)); + world.Advance(1); + Assert.Equal(2_500, world.AuraArmorBonusBp(grunt.Id)); + + grunt.FindModule()!.SetDestination(At(30, 0)); + // Walking out of the 5-unit radius: dropped on a later rescan + // (RecomputeTicks = 4), and the dead banner case never leaks because + // the table is rebuilt from live carriers every tick. + world.Advance(80); + Assert.Equal(At(30, 0), grunt.Position); + Assert.Equal(0, world.AuraArmorBonusBp(grunt.Id)); + } + + // ---- serialization + determinism ------------------------------------------ + + private static SimWorld BuildCombinedScenario(ulong seed = 21) + { + var world = NewWorld(seed); + world.AddTeamResources(0, 2000); + world.AddTeamResources(1, 2000); + var keep = world.SpawnObject("keep", 0, FixedVector2.Zero); // id 1 + world.SpawnObject("banner", 0, At(5, 0)); // id 2 + var grunt = world.SpawnObject("grunt", 0, At(4, 0)); // id 3 + world.SpawnObject("grunt", 0, At(6, 0)); // id 4 + var horde = world.SpawnObject("horde", 1, At(20, 0)); // id 5 + var target = world.SpawnObject("target", 1, At(8, 0)); // id 6 + var arrow = world.SpawnObject("arrow", 1, At(20, 5)); // id 7 + arrow.FindModule()!.Launch(world, arrow, grunt.Id); + + world.SubmitCommand(TestWorlds.Command(10, 0, 0, "queue_production", + ("id", CommandValue.OfLong(keep.Id)), ("template", CommandValue.OfString("soldier")))); + world.SubmitCommand(TestWorlds.Command(10, 0, 1, "queue_production", + ("id", CommandValue.OfLong(keep.Id)), ("template", CommandValue.OfString("soldier")))); + world.SubmitCommand(TestWorlds.Command(12, 0, 2, "cancel_production", + ("id", CommandValue.OfLong(keep.Id)), ("index", CommandValue.OfLong(1)))); + world.SubmitCommand(TestWorlds.Command(40, 0, 3, "move", + ("id", CommandValue.OfLong(grunt.Id)), ("x", CommandValue.OfFixed(Fixed64.FromInt(25))), ("y", CommandValue.OfFixed(Fixed64.Zero)))); + world.SubmitCommand(TestWorlds.Command(50, 1, 0, "damage", + ("id", CommandValue.OfLong(horde.Id)), ("amount", CommandValue.OfLong(250)))); + world.SubmitCommand(TestWorlds.Command(60, 0, 4, "damage", + ("id", CommandValue.OfLong(target.Id)), ("amount", CommandValue.OfLong(120)))); + world.SubmitCommand(TestWorlds.Command(400, 1, 1, "damage", + ("id", CommandValue.OfLong(horde.Id)), ("amount", CommandValue.OfLong(100)))); + return world; + } + + [Fact] + public void SnapshotRoundTripsMidFlightMidAuraAndMidQueue() + { + var original = BuildCombinedScenario(); + original.Advance(15); // queue mid-build (5 of 20 ticks), aura active, cancel already refunded + var restored = SimWorld.Restore(original.Snapshot(), Config(21), ModuleRegistry.CreateDefault()); + Assert.Equal(original.StateHash(), restored.StateHash()); + + // The aura table is derived state (not serialized): prove the restored + // world rebuilt it by pushing identical damage through both worlds. + Assert.Equal(original.AuraArmorBonusBp(3), restored.AuraArmorBonusBp(3)); + foreach (var world in new[] { original, restored }) + { + world.SubmitCommand(TestWorlds.Command(16, 1, 9, "damage", + ("id", CommandValue.OfLong(3)), ("amount", CommandValue.OfLong(100)))); + } + for (var tick = 16; tick <= 100; tick++) + { + original.Tick(); + restored.Tick(); + Assert.Equal(original.StateHash(), restored.StateHash()); + } + } + + [Fact] + public void SnapshotRoundTripsWithProjectileLiterallyMidFlight() + { + var original = NewWorld(33); + var target = original.SpawnObject("target", 1, At(8, 0)); + var arrow = original.SpawnObject("arrow", 0, FixedVector2.Zero); + arrow.FindModule()!.Launch(original, arrow, target.Id); + original.Advance(3); // 3 of 8 flight ticks flown + + var restored = SimWorld.Restore(original.Snapshot(), Config(33), ModuleRegistry.CreateDefault()); + Assert.Equal(original.StateHash(), restored.StateHash()); + for (var tick = 0; tick < 10; tick++) + { + original.Tick(); + restored.Tick(); + Assert.Equal(original.StateHash(), restored.StateHash()); + } + Assert.Equal(180, target.FindModule()!.Health); + Assert.Single(restored.Objects); + } + + [Fact] + public void CombinedScenarioTwinRunsStayHashIdenticalFor1000Ticks() + { + var a = BuildCombinedScenario(); + var b = BuildCombinedScenario(); + for (var tick = 1; tick <= 1000; tick++) + { + a.Tick(); + b.Tick(); + Assert.Equal(a.StateHash(), b.StateHash()); + } + // Sanity: production ran (one soldier built, one cancelled+refunded), + // the horde died at tick 400, the aura outlived the run. + Assert.Contains(a.Objects.Values, o => o.TemplateName == "soldier"); + Assert.DoesNotContain(a.Objects.Values, o => o.TemplateName == "horde"); + Assert.Contains(a.Objects.Values, o => o.TemplateName == "banner"); + } + + [Fact] + public void NewModuleTypesAreRegistered() + { + var registry = ModuleRegistry.CreateDefault(); + foreach (var typeName in new[] + { + HordeContainModule.TypeName, BezierProjectileModule.TypeName, AttributeModifierAuraModule.TypeName, + }) + { + Assert.True(registry.TryCreate(new ModuleSpec(typeName, null), out _), $"{typeName} not registered"); + } + } +} diff --git a/engine/OpenBfme.Sim.Tests/CombatTests.cs b/engine/OpenBfme.Sim.Tests/CombatTests.cs new file mode 100644 index 0000000..3ff0d43 --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/CombatTests.cs @@ -0,0 +1,170 @@ +using OpenBfme.Sim; +using Xunit; + +namespace OpenBfme.Sim.Tests; + +public class CombatTests +{ + private static ModuleSpec Body(long maxHealth) => + new(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = maxHealth }); + + private static ModuleSpec Mover() => + new(LinearMoverModule.TypeName, new Dictionary + { + ["SpeedPerTickRaw"] = Fixed64.FromFraction(1, 5).Raw, + }); + + private static ModuleSpec Weapon(long damage, string damageType = DamageTypes.Slash, int reload = 8, int range = 2) => + new(WeaponModule.TypeName, + new Dictionary + { + ["Damage"] = damage, + ["ReloadTicks"] = reload, + ["RangeRaw"] = Fixed64.FromInt(range).Raw, + }, + new Dictionary { ["DamageType"] = damageType }); + + private static ModuleSpec Ai() => + new(AiCombatModule.TypeName, new Dictionary + { + ["VisionRangeRaw"] = Fixed64.FromInt(30).Raw, + }); + + private static SimConfig Config() => new( + new[] + { + new ObjectTemplate("swordsman", new[] { Body(100), Mover(), Weapon(12), Ai() }), + new ObjectTemplate("veteran", new[] + { + Body(100), + new ModuleSpec(ArmorModule.TypeName, new Dictionary + { + ["Armor:slash"] = 5_000, + ["ArmorDefault"] = 10_000, + }), + Mover(), Weapon(12), Ai(), + }), + new ObjectTemplate("peasant_crushable", new[] + { + Body(60), + new ModuleSpec(SquishCollideModule.TypeName, null), + }), + new ObjectTemplate("stone_wall", new[] { Body(60) }), + new ObjectTemplate("knight", new[] { Body(200), Mover(), Weapon(30, DamageTypes.Crush, 6, 1), Ai() }), + }, + randomSeed: 11, + teamCount: 2); + + private static SimWorld NewWorld() => new(Config(), ModuleRegistry.CreateDefault()); + + [Fact] + public void OpposingSoldiersFightToDeterministicResolution() + { + SimWorld Build() + { + var world = NewWorld(); + world.SpawnObject("swordsman", 0, new FixedVector2(Fixed64.FromInt(-6), Fixed64.Zero)); + world.SpawnObject("swordsman", 1, new FixedVector2(Fixed64.FromInt(6), Fixed64.Zero)); + return world; + } + + var a = Build(); + var b = Build(); + var resolved = -1; + for (var tick = 1; tick <= 600; tick++) + { + a.Tick(); + b.Tick(); + Assert.Equal(a.StateHash(), b.StateHash()); + if (resolved < 0 && a.Objects.Count == 1) + { + resolved = tick; + } + } + Assert.True(resolved > 0, "combat never resolved"); + // Symmetric duel: team 0's unit has the lower id, scans first, so the + // survivor must be deterministic — assert it IS object 1. + Assert.Equal(1, a.Objects.Values.Single().Id); + } + + [Fact] + public void ArmorHalvesSlashDamage() + { + var world = NewWorld(); + var veteran = world.SpawnObject("veteran", 0, FixedVector2.Zero); + var plain = world.SpawnObject("swordsman", 0, FixedVector2.Zero); + world.DealDamage(veteran, 12, DamageTypes.Slash); + world.DealDamage(plain, 12, DamageTypes.Slash); + Assert.Equal(94, veteran.FindModule()!.Health); + Assert.Equal(88, plain.FindModule()!.Health); + // Non-slash uses ArmorDefault (100%). + world.DealDamage(veteran, 10, DamageTypes.Siege); + Assert.Equal(84, veteran.FindModule()!.Health); + } + + [Fact] + public void CrushOnlyLandsOnCrushableTargets() + { + var world = NewWorld(); + var peasant = world.SpawnObject("peasant_crushable", 1, FixedVector2.Zero); + var wall = world.SpawnObject("stone_wall", 1, FixedVector2.Zero); + world.DealDamage(peasant, 30, DamageTypes.Crush); + world.DealDamage(wall, 30, DamageTypes.Crush); + Assert.Equal(30, peasant.FindModule()!.Health); + Assert.Equal(60, wall.FindModule()!.Health); + } + + [Fact] + public void KnightRidesDownCrushablePeasant() + { + var world = NewWorld(); + world.SpawnObject("knight", 0, new FixedVector2(Fixed64.FromInt(-8), Fixed64.Zero)); + var peasant = world.SpawnObject("peasant_crushable", 1, new FixedVector2(Fixed64.FromInt(4), Fixed64.Zero)); + world.Advance(400); + Assert.True(peasant.IsDead || !world.Objects.ContainsKey(peasant.Id), "peasant should be crushed"); + Assert.Single(world.Objects); + } + + [Fact] + public void MidBattleSnapshotRestoresToIdenticalOutcome() + { + var world = NewWorld(); + world.SpawnObject("veteran", 0, new FixedVector2(Fixed64.FromInt(-5), Fixed64.Zero)); + world.SpawnObject("swordsman", 1, new FixedVector2(Fixed64.FromInt(5), Fixed64.Zero)); + world.Advance(60); + + var restored = SimWorld.Restore(world.Snapshot(), Config(), ModuleRegistry.CreateDefault()); + for (var tick = 61; tick <= 500; tick++) + { + world.Tick(); + restored.Tick(); + Assert.Equal(world.StateHash(), restored.StateHash()); + } + Assert.Equal(world.Objects.Count, restored.Objects.Count); + } + + [Fact] + public void DeadTargetsAreDroppedAndRetargetingIsDeterministic() + { + SimWorld Build() + { + var world = NewWorld(); + world.SpawnObject("swordsman", 0, new FixedVector2(Fixed64.FromInt(-4), Fixed64.Zero)); + world.SpawnObject("swordsman", 1, new FixedVector2(Fixed64.FromInt(4), Fixed64.Zero)); + world.SpawnObject("swordsman", 1, new FixedVector2(Fixed64.FromInt(5), Fixed64.FromInt(1))); + return world; + } + + var a = Build(); + var b = Build(); + for (var tick = 1; tick <= 900; tick++) + { + a.Tick(); + b.Tick(); + Assert.Equal(a.StateHash(), b.StateHash()); + } + // 2v1: team 1 must win and both survivors belong to team 1. + Assert.All(a.Objects.Values, o => Assert.Equal(1, o.Team)); + Assert.True(a.Objects.Count >= 1); + } +} diff --git a/engine/OpenBfme.Sim.Tests/CrossPlatformPinTests.cs b/engine/OpenBfme.Sim.Tests/CrossPlatformPinTests.cs new file mode 100644 index 0000000..9abe6f4 --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/CrossPlatformPinTests.cs @@ -0,0 +1,30 @@ +using OpenBfme.Sim; +using Xunit; + +namespace OpenBfme.Sim.Tests; + +/// +/// Pins the state hash of a fixed scenario to a constant. CI runs this suite on +/// Windows AND Linux: both must reproduce the identical SHA-256, proving the +/// fixed-point kernel is bit-identical across platforms — the property the +/// 8-player lockstep target depends on. If an intentional sim change moves the +/// hash, re-pin it in the same commit and say so in the commit message. +/// +public class CrossPlatformPinTests +{ + public const string PinnedHash = "f1b7c86cedf7989f81e0838e07cd5660cbe5baddd9602d1ff43489f7d1dfa41e"; + + private static string ComputeScenarioHash() + { + var world = TestWorlds.BuildAndSubmit(); + world.Advance(2000); + return world.StateHash(); + } + + [Fact] + public void FixedScenarioHashMatchesPinnedConstant() + { + var actual = ComputeScenarioHash(); + Assert.True(PinnedHash == actual, $"pinned={PinnedHash} actual={actual}"); + } +} diff --git a/engine/OpenBfme.Sim.Tests/DeathAndBodyModuleTests.cs b/engine/OpenBfme.Sim.Tests/DeathAndBodyModuleTests.cs new file mode 100644 index 0000000..3b0d0be --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/DeathAndBodyModuleTests.cs @@ -0,0 +1,135 @@ +using OpenBfme.Sim; +using Xunit; + +namespace OpenBfme.Sim.Tests; + +public class DeathAndBodyModuleTests +{ + private static SimConfig Config() => new( + new[] + { + new ObjectTemplate("immortal_shrine", new[] + { + new ModuleSpec(ImmortalBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 50 }), + }), + new ObjectTemplate("summon", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 60 }), + new ModuleSpec(LifetimeModule.TypeName, new Dictionary { ["LifetimeTicks"] = 40 }), + new ModuleSpec(DestroyDieModule.TypeName, null), + }), + new ObjectTemplate("tower", new[] + { + new ModuleSpec(StructureBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 300 }), + new ModuleSpec(StructureCollapseModule.TypeName, new Dictionary { ["CollapseTicks"] = 12 }), + }), + new ObjectTemplate("monument", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 10 }), + new ModuleSpec(KeepObjectDieModule.TypeName, null), + }), + }, + randomSeed: 7, + teamCount: 2); + + private static SimWorld NewWorld() => new(Config(), ModuleRegistry.CreateDefault()); + + [Fact] + public void ImmortalBodySurvivesArbitraryDamageAtOneHealth() + { + var world = NewWorld(); + var shrine = world.SpawnObject("immortal_shrine", 0, FixedVector2.Zero); + world.DealDamage(shrine, 10_000); + world.Advance(5); + Assert.Single(world.Objects); + Assert.Equal(1, shrine.FindModule()!.Health); + world.DealDamage(shrine, 10_000); + Assert.Equal(1, shrine.FindModule()!.Health); + } + + [Fact] + public void LifetimeExpiresThroughDeathPipelineOnExactTick() + { + var world = NewWorld(); + world.SpawnObject("summon", 0, FixedVector2.Zero); + world.Advance(39); + Assert.Single(world.Objects); + world.Advance(1); + Assert.Empty(world.Objects); + } + + [Fact] + public void StructureCollapseHoldsRubbleThenRemoves() + { + var world = NewWorld(); + var tower = world.SpawnObject("tower", 1, FixedVector2.Zero); + world.DealDamage(tower, 300); + Assert.True(tower.IsDying); + world.Advance(11); + Assert.Single(world.Objects); + world.Advance(1); + Assert.Empty(world.Objects); + } + + [Fact] + public void KeepObjectDieCorpsePersistsIndefinitely() + { + var world = NewWorld(); + var monument = world.SpawnObject("monument", 0, FixedVector2.Zero); + world.DealDamage(monument, 10); + world.Advance(500); + Assert.Single(world.Objects); + Assert.True(monument.IsDying); + } + + [Fact] + public void MixedDeathScenarioIsDeterministicAndSnapshotSafe() + { + SimWorld Build() + { + var world = NewWorld(); + world.SpawnObject("immortal_shrine", 0, FixedVector2.Zero); + world.SpawnObject("summon", 0, new FixedVector2(Fixed64.FromInt(1), Fixed64.Zero)); + world.SpawnObject("tower", 1, new FixedVector2(Fixed64.FromInt(2), Fixed64.Zero)); + world.SpawnObject("monument", 1, new FixedVector2(Fixed64.FromInt(3), Fixed64.Zero)); + world.SubmitCommand(TestWorlds.Command(5, 1, 0, "damage", + ("id", CommandValue.OfLong(3)), ("amount", CommandValue.OfLong(300)))); + world.SubmitCommand(TestWorlds.Command(6, 0, 1, "damage", + ("id", CommandValue.OfLong(4)), ("amount", CommandValue.OfLong(10)))); + return world; + } + + var a = Build(); + var b = Build(); + a.Advance(10); + b.Advance(10); + var restored = SimWorld.Restore(a.Snapshot(), Config(), ModuleRegistry.CreateDefault()); + for (var tick = 11; tick <= 80; tick++) + { + a.Tick(); + b.Tick(); + restored.Tick(); + Assert.Equal(a.StateHash(), b.StateHash()); + Assert.Equal(a.StateHash(), restored.StateHash()); + } + // Survivors: shrine (immortal) + monument corpse (kept). Summon expired, tower collapsed. + Assert.Equal(2, a.Objects.Count); + } + + [Fact] + public void RegistryCoversElevenModuleTypes() + { + var registry = ModuleRegistry.CreateDefault(); + var typeNames = new[] + { + ActiveBodyModule.TypeName, ResourceGeneratorModule.TypeName, LinearMoverModule.TypeName, + SlowDeathModule.TypeName, ProductionModule.TypeName, GettingBuiltModule.TypeName, + StructureBodyModule.TypeName, ImmortalBodyModule.TypeName, LifetimeModule.TypeName, + DestroyDieModule.TypeName, KeepObjectDieModule.TypeName, StructureCollapseModule.TypeName, + }; + foreach (var typeName in typeNames) + { + Assert.True(registry.TryCreate(new ModuleSpec(typeName, null), out _), $"{typeName} not registered"); + } + } +} diff --git a/engine/OpenBfme.Sim.Tests/DeterminismTests.cs b/engine/OpenBfme.Sim.Tests/DeterminismTests.cs new file mode 100644 index 0000000..2d7f737 --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/DeterminismTests.cs @@ -0,0 +1,291 @@ +using OpenBfme.Sim; +using Xunit; + +namespace OpenBfme.Sim.Tests; + +public static class TestWorlds +{ + public static SimConfig Config(ulong seed = 42) => new( + new[] + { + new ObjectTemplate("soldier", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 120 }), + new ModuleSpec(LinearMoverModule.TypeName, new Dictionary + { + ["SpeedPerTickRaw"] = Fixed64.FromFraction(1, 4).Raw, + }), + }), + new ObjectTemplate("farm", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 500 }), + new ModuleSpec(ResourceGeneratorModule.TypeName, new Dictionary + { + ["IntervalTicks"] = 15, + ["Amount"] = 5, + }), + }), + new ObjectTemplate("mystery", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 10 }), + new ModuleSpec("NotYetImplementedBehavior", null), + }), + }, + seed, + teamCount: 2); + + public static List ScriptedCommands() + { + Fixed64 F(long n, long d = 1) => Fixed64.FromFraction(n, d); + var commands = new List + { + Command(1, 0, 0, "spawn", ("template", CommandValue.OfString("farm")), ("x", CommandValue.OfFixed(F(-20))), ("y", CommandValue.OfFixed(F(-10)))), + Command(1, 1, 0, "spawn", ("template", CommandValue.OfString("farm")), ("x", CommandValue.OfFixed(F(20))), ("y", CommandValue.OfFixed(F(10)))), + Command(2, 0, 1, "spawn", ("template", CommandValue.OfString("soldier")), ("x", CommandValue.OfFixed(F(-18))), ("y", CommandValue.OfFixed(F(-9)))), + Command(2, 1, 1, "spawn", ("template", CommandValue.OfString("soldier")), ("x", CommandValue.OfFixed(F(18))), ("y", CommandValue.OfFixed(F(9)))), + Command(5, 0, 2, "move", ("id", CommandValue.OfLong(3)), ("x", CommandValue.OfFixed(F(15))), ("y", CommandValue.OfFixed(F(7)))), + Command(5, 1, 2, "move", ("id", CommandValue.OfLong(4)), ("x", CommandValue.OfFixed(F(-15, 1))), ("y", CommandValue.OfFixed(F(-71, 10)))), + Command(40, 0, 3, "roll"), + Command(40, 1, 3, "roll"), + Command(200, 0, 4, "damage", ("id", CommandValue.OfLong(4)), ("amount", CommandValue.OfLong(80))), + Command(900, 1, 5, "damage", ("id", CommandValue.OfLong(3)), ("amount", CommandValue.OfLong(200))), + Command(1600, 0, 6, "spawn", ("template", CommandValue.OfString("soldier")), ("x", CommandValue.OfFixed(F(0))), ("y", CommandValue.OfFixed(F(0)))), + Command(1610, 0, 7, "move", ("id", CommandValue.OfLong(5)), ("x", CommandValue.OfFixed(F(30))), ("y", CommandValue.OfFixed(F(-30)))), + Command(2500, 1, 8, "roll"), + }; + return commands; + } + + public static SimCommand Command(int tick, int team, int seq, string type, + params (string Key, CommandValue Value)[] args) => + new(tick, team, seq, type, args.Select(a => new KeyValuePair(a.Key, a.Value))); + + public static SimWorld BuildAndSubmit(IEnumerable? commands = null, ulong seed = 42) + { + var world = new SimWorld(Config(seed), ModuleRegistry.CreateDefault()); + foreach (var command in commands ?? ScriptedCommands()) + { + Assert.True(world.SubmitCommand(command), $"command {command.Type}@{command.Tick} rejected"); + } + return world; + } +} + +public class DeterminismTests +{ + private const int ProofTicks = 3000; + + [Fact] + public void TwinRunsStayHashIdenticalForEveryTick() + { + var a = TestWorlds.BuildAndSubmit(); + var b = TestWorlds.BuildAndSubmit(); + for (var tick = 1; tick <= ProofTicks; tick++) + { + a.Tick(); + b.Tick(); + Assert.Equal(a.StateHash(), b.StateHash()); + } + Assert.True(a.Objects.Count > 0, "scenario should leave survivors"); + } + + [Fact] + public void HashIsSensitiveToASingleAlteredCommand() + { + var commands = TestWorlds.ScriptedCommands(); + var altered = TestWorlds.ScriptedCommands(); + altered[4] = TestWorlds.Command(5, 0, 2, "move", + ("id", CommandValue.OfLong(3)), + ("x", CommandValue.OfFixed(Fixed64.FromFraction(151, 10))), + ("y", CommandValue.OfFixed(Fixed64.FromInt(7)))); + + var a = TestWorlds.BuildAndSubmit(commands); + var b = TestWorlds.BuildAndSubmit(altered); + // The altered unit is destroyed later in the script, so the worlds may + // legitimately reconverge; sensitivity means the hash differs while the + // divergent state exists. + var diverged = false; + for (var tick = 1; tick <= ProofTicks; tick++) + { + a.Tick(); + b.Tick(); + if (a.StateHash() != b.StateHash()) + { + diverged = true; + } + } + Assert.True(diverged, "altered command never changed the state hash"); + } + + [Fact] + public void CommandSubmissionOrderDoesNotAffectOutcome() + { + var forward = TestWorlds.ScriptedCommands(); + var reversed = TestWorlds.ScriptedCommands(); + reversed.Reverse(); + + var a = TestWorlds.BuildAndSubmit(forward); + var b = TestWorlds.BuildAndSubmit(reversed); + a.Advance(ProofTicks); + b.Advance(ProofTicks); + Assert.Equal(a.StateHash(), b.StateHash()); + } + + [Fact] + public void SnapshotRoundTripsMidRunAndStaysIdentical() + { + var original = TestWorlds.BuildAndSubmit(); + original.Advance(1500); + + var restored = SimWorld.Restore(original.Snapshot(), TestWorlds.Config(), ModuleRegistry.CreateDefault()); + Assert.Equal(original.StateHash(), restored.StateHash()); + + for (var tick = 1501; tick <= ProofTicks; tick++) + { + original.Tick(); + restored.Tick(); + Assert.Equal(original.StateHash(), restored.StateHash()); + } + } + + [Fact] + public void SnapshotOfRestoredWorldIsByteIdentical() + { + var original = TestWorlds.BuildAndSubmit(); + original.Advance(777); + var snapshot = original.Snapshot(); + var restored = SimWorld.Restore(snapshot, TestWorlds.Config(), ModuleRegistry.CreateDefault()); + Assert.Equal(snapshot, restored.Snapshot()); + } + + [Fact] + public void RandomStreamIsAuthoritativeState() + { + var withRoll = TestWorlds.BuildAndSubmit(); + var withoutRoll = TestWorlds.BuildAndSubmit( + TestWorlds.ScriptedCommands().Where(c => c.Type != "roll").ToList()); + withRoll.Advance(50); + withoutRoll.Advance(50); + Assert.NotEqual(withRoll.StateHash(), withoutRoll.StateHash()); + } + + [Fact] + public void UnimplementedModuleTypesAreRecordedAsGaps() + { + var world = new SimWorld(TestWorlds.Config(), ModuleRegistry.CreateDefault()); + world.SpawnObject("mystery", 0, FixedVector2.Zero); + world.SpawnObject("mystery", 1, FixedVector2.Zero); + Assert.Equal(2, world.ModuleGaps["NotYetImplementedBehavior"]); + } + + [Fact] + public void CommandsForPastTicksAreRejected() + { + var world = new SimWorld(TestWorlds.Config(), ModuleRegistry.CreateDefault()); + world.Advance(10); + Assert.False(world.SubmitCommand(TestWorlds.Command(10, 0, 0, "roll"))); + Assert.False(world.SubmitCommand(TestWorlds.Command(3, 0, 0, "roll"))); + Assert.True(world.SubmitCommand(TestWorlds.Command(11, 0, 0, "roll"))); + } + + [Fact] + public void DeadObjectsAreRemovedAndIncomeStops() + { + var world = new SimWorld(TestWorlds.Config(), ModuleRegistry.CreateDefault()); + var farm = world.SpawnObject("farm", 0, FixedVector2.Zero); + world.Advance(15); + var incomeAfter15 = world.TeamResources(0); + Assert.Equal(5, incomeAfter15); + world.DealDamage(farm, 500); + world.Advance(1); + Assert.Empty(world.Objects); + world.Advance(60); + Assert.Equal(incomeAfter15, world.TeamResources(0)); + } + + [Fact] + public void MoverArrivesExactlyAtDestination() + { + var world = new SimWorld(TestWorlds.Config(), ModuleRegistry.CreateDefault()); + var soldier = world.SpawnObject("soldier", 0, FixedVector2.Zero); + var destination = new FixedVector2(Fixed64.FromInt(3), Fixed64.FromInt(4)); + soldier.FindModule()!.SetDestination(destination); + world.Advance(200); + Assert.Equal(destination, soldier.Position); + } +} + +public class Fixed64Tests +{ + [Fact] + public void FractionRoundTripsThroughArithmetic() + { + var quarter = Fixed64.FromFraction(1, 4); + Assert.Equal(Fixed64.One, quarter + quarter + quarter + quarter); + Assert.Equal(Fixed64.FromInt(1), Fixed64.FromFraction(4, 4)); + Assert.Equal(Fixed64.FromFraction(3, 8), quarter * Fixed64.FromFraction(3, 2)); + } + + [Fact] + public void SqrtIsExactForPerfectSquaresAndMonotonic() + { + Assert.Equal(Fixed64.FromInt(12), Fixed64.Sqrt(Fixed64.FromInt(144))); + Assert.Equal(Fixed64.Zero, Fixed64.Sqrt(Fixed64.Zero)); + var previous = Fixed64.Zero; + for (var i = 1; i <= 500; i++) + { + var root = Fixed64.Sqrt(Fixed64.FromInt(i)); + Assert.True(root >= previous, $"sqrt not monotonic at {i}"); + Assert.True(root * root <= Fixed64.FromInt(i), $"sqrt overshoots at {i}"); + previous = root; + } + } + + [Fact] + public void DivisionByZeroAndOverflowThrow() + { + Assert.Throws(() => Fixed64.One / Fixed64.Zero); + Assert.Throws(() => Fixed64.MaxValue + Fixed64.One); + Assert.Throws(() => Fixed64.MaxValue * Fixed64.FromInt(2)); + } +} + +public class DeterministicRandomTests +{ + [Fact] + public void SameSeedSameSequence() + { + var a = new DeterministicRandom(1234, 7); + var b = new DeterministicRandom(1234, 7); + for (var i = 0; i < 1000; i++) + { + Assert.Equal(a.NextUInt32(), b.NextUInt32()); + } + } + + [Fact] + public void SerializedStreamContinuesIdentically() + { + var original = new DeterministicRandom(99); + for (var i = 0; i < 137; i++) + { + original.NextUInt32(); + } + var (state, increment) = original.Serialize(); + var restored = DeterministicRandom.Deserialize(state, increment); + for (var i = 0; i < 1000; i++) + { + Assert.Equal(original.NextUInt32(), restored.NextUInt32()); + } + } + + [Fact] + public void NextBelowStaysInRange() + { + var random = new DeterministicRandom(5); + for (var i = 0; i < 10000; i++) + { + Assert.InRange(random.NextBelow(7), 0u, 6u); + } + } +} diff --git a/engine/OpenBfme.Sim.Tests/DualRunOracleTests.cs b/engine/OpenBfme.Sim.Tests/DualRunOracleTests.cs new file mode 100644 index 0000000..64f23ee --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/DualRunOracleTests.cs @@ -0,0 +1,343 @@ +using OpenBfme.Sim; +using Xunit; +using Xunit.Abstractions; + +namespace OpenBfme.Sim.Tests; + +/// +/// Tier-1 dual-run oracle: replays the GDScript retail sim's recorded scenario +/// (game/tests/retail_dualrun_trace_runner.gd) inside the C# SimWorld and +/// compares SEMANTIC metrics per sampled tick. +/// +/// HONEST SCOPE — the engine implements a small module subset, so only the +/// genuine overlap gates: +/// * team resources per sampled tick: EXACT (gating; production cost is now +/// engine-native — the barracks template carries "Cost:soldier" and the +/// ProductionModule debits it at the command tick, matching the trace's +/// "queue_unit charges cost at the command tick"), +/// * mobile entity counts per sampled tick: EXACT (gating; spawn phase now +/// matches retail exactly — both sims spawn the battalion during +/// command_tick + build_ticks, the former 1-tick offset is closed), +/// * positions: tolerance-compared and REPORTED but non-gating, because the +/// integrators genuinely differ today — the retail sim accelerates +/// (current_speed += accel*dt, brakes on the final leg) while +/// LinearMoverModule moves at constant speed; exact parity is a P2 +/// locomotor work item, not a scenario-construction choice. +/// +/// Scenario-construction mirrors (documented, deliberately NOT engine edits): +/// * the recorded out-of-band resource grant is applied at the same +/// after-tick/after-sample boundary the GDScript harness used, +/// * the produced battalion is excluded from position comparison: the +/// retail sim walks it door -> create point -> rally; the engine now has +/// an exit->rally walk (ProductionModule RallyX/YRaw) but not the retail +/// door->create-point leg, so positions stay excluded for produced units. +/// +public class DualRunOracleTests +{ + private const string TraceRelativePath = ".private/retail-work/reports/dualrun/gdscript_trace.json"; + private const string FarmTemplate = "farm"; + private const string BarracksTemplate = "barracks"; + private const string BattalionTemplate = "soldier"; + private const double PositionToleranceUnits = 1.5; + + private readonly ITestOutputHelper _output; + + public DualRunOracleTests(ITestOutputHelper output) => _output = output; + + [Fact] + public void ReplayMatchesGdScriptTraceOnTheSupportedOverlap() + { + var tracePath = LocateTrace(); + if (tracePath == null) + { + _output.WriteLine($"SKIPPED: no trace at {TraceRelativePath}; run game/tests/retail_dualrun_trace_runner.gd first."); + return; + } + var trace = DualRunTrace.Load(tracePath); + + var world = BuildWorld(trace, out var idMap, out var producedTraceIds); + SubmitReplayCommands(trace, world, idMap); + + // Out-of-band ledger adjustments applied after the given tick's sample: + // only the recorded grant remains — production cost is engine-native now. + var adjustmentsAfterTick = new Dictionary(); + foreach (var command in trace.Commands) + { + if (command.Type == "grant_resources") + { + adjustmentsAfterTick[command.Tick] = + adjustmentsAfterTick.GetValueOrDefault(command.Tick) + command.Amount; + } + } + + var samplesByTick = trace.Samples.ToDictionary(sample => sample.Tick); + var divergences = new DivergenceReport(); + CompareSample(trace, samplesByTick[0], world, idMap, producedTraceIds, divergences); + for (var tick = 1; tick <= trace.TotalTicks; tick++) + { + world.Tick(); + if (samplesByTick.TryGetValue(tick, out var sample)) + { + CompareSample(trace, sample, world, idMap, producedTraceIds, divergences); + } + if (adjustmentsAfterTick.TryGetValue(tick, out var delta)) + { + world.AddTeamResources(0, delta); + } + } + + _output.WriteLine($"trace: {tracePath}"); + _output.WriteLine($"sampled ticks compared: {divergences.SamplesCompared} (interval {trace.SampleInterval}, total {trace.TotalTicks})"); + _output.WriteLine($"resources: first divergence tick = {Describe(divergences.FirstResourceDivergenceTick)}, divergent samples = {divergences.ResourceDivergences}"); + _output.WriteLine($"entity counts: first divergence tick = {Describe(divergences.FirstCountDivergenceTick)}, divergent samples = {divergences.CountDivergences}"); + _output.WriteLine($"positions (non-gating, tolerance {PositionToleranceUnits}): first out-of-tolerance tick = {Describe(divergences.FirstPositionDivergenceTick)}, out-of-tolerance samples = {divergences.PositionDivergences}, max gap = {divergences.MaxPositionGap:0.###} units"); + _output.WriteLine($"health (non-gating): first divergence tick = {Describe(divergences.FirstHealthDivergenceTick)}, divergent samples = {divergences.HealthDivergences}"); + foreach (var line in divergences.Details.Take(20)) + { + _output.WriteLine(line); + } + + Assert.True(divergences.ResourceDivergences == 0, + $"team resources diverged first at sampled tick {divergences.FirstResourceDivergenceTick}"); + Assert.True(divergences.CountDivergences == 0, + $"entity counts diverged first at sampled tick {divergences.FirstCountDivergenceTick}"); + } + + private static SimWorld BuildWorld(DualRunTrace trace, out Dictionary idMap, out HashSet producedTraceIds) + { + var farm = trace.Structures.Single(s => s.IncomePerPayout > 0); + var barracks = trace.Structures.Single(s => s.IncomePerPayout == 0); + var templates = new[] + { + new ObjectTemplate(FarmTemplate, new[] + { + new ModuleSpec(StructureBodyModule.TypeName, new Dictionary { ["MaxHealth"] = farm.MaximumHealth }), + new ModuleSpec(ResourceGeneratorModule.TypeName, new Dictionary + { + ["IntervalTicks"] = trace.FarmPayoutIntervalTicks, + ["Amount"] = trace.FarmIncomePerPayout, + }), + }), + new ObjectTemplate(BarracksTemplate, new[] + { + new ModuleSpec(StructureBodyModule.TypeName, new Dictionary { ["MaxHealth"] = barracks.MaximumHealth }), + new ModuleSpec(ProductionModule.TypeName, new Dictionary + { + ["Build:" + BattalionTemplate] = trace.UnitBuildTicks, + ["Cost:" + BattalionTemplate] = trace.UnitCost, + }), + }), + new ObjectTemplate(BattalionTemplate, new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary + { + ["MaxHealth"] = checked(trace.UnitMemberHealth * trace.UnitMemberCount), + }), + new ModuleSpec(LinearMoverModule.TypeName, new Dictionary + { + ["SpeedPerTickRaw"] = trace.SpeedPerTick().Raw, + }), + }), + }; + var world = new SimWorld(new SimConfig(templates, randomSeed: 1, teamCount: 2), ModuleRegistry.CreateDefault()); + world.AddTeamResources(0, trace.StartingResources); + world.AddTeamResources(1, trace.StartingResources); + + // Spawn the initial battalions in trace-id order, then the structures; + // the map from GDScript ids to engine ids is positional. + idMap = new Dictionary(); + foreach (var entity in trace.InitialEntities.OrderBy(e => e.Id)) + { + var spawned = world.SpawnObject(BattalionTemplate, entity.Team, new FixedVector2(entity.X, entity.Y)); + idMap.Add(entity.Id, spawned.Id); + } + foreach (var structure in trace.Structures.OrderBy(s => s.Id)) + { + var template = structure.IncomePerPayout > 0 ? FarmTemplate : BarracksTemplate; + var spawned = world.SpawnObject(template, structure.Team, new FixedVector2(structure.X, structure.Y)); + idMap.Add(structure.Id, spawned.Id); + } + producedTraceIds = new HashSet(); + return world; + } + + private static void SubmitReplayCommands(DualRunTrace trace, SimWorld world, Dictionary idMap) + { + foreach (var command in trace.Commands) + { + switch (command.Type) + { + case "issue_move": + foreach (var targetId in command.TargetIds!) + { + Assert.True(world.SubmitCommand(TestWorlds.Command(command.Tick, command.Team, command.Seq, "move", + ("id", CommandValue.OfLong(idMap[targetId])), + ("x", CommandValue.OfFixed(command.DestinationX)), + ("y", CommandValue.OfFixed(command.DestinationY))))); + } + break; + case "queue_unit": + Assert.Equal(trace.UnitType, command.UnitType); + Assert.True(world.SubmitCommand(TestWorlds.Command(command.Tick, command.Team, command.Seq, "queue_production", + ("id", CommandValue.OfLong(idMap[command.ProducerId])), + ("template", CommandValue.OfString(BattalionTemplate))))); + break; + case "grant_resources": + break; // handled as an after-tick ledger adjustment + default: + Assert.Fail($"trace command type '{command.Type}' has no replay mapping"); + break; + } + } + } + + private void CompareSample( + DualRunTrace trace, + TraceSample sample, + SimWorld world, + Dictionary idMap, + HashSet producedTraceIds, + DivergenceReport divergences) + { + divergences.SamplesCompared++; + + foreach (var (team, recorded) in sample.TeamResources) + { + var replayed = world.TeamResources(team); + if (replayed != recorded) + { + divergences.RecordResources(sample.Tick, + $" tick {sample.Tick}: team {team} resources gd={recorded} cs={replayed}"); + } + } + + var mobileObjects = world.Objects.Values + .Where(o => o.TemplateName == BattalionTemplate) + .OrderBy(o => o.Id) + .ToList(); + if (mobileObjects.Count != sample.EntityCount) + { + divergences.RecordCount(sample.Tick, + $" tick {sample.Tick}: entity count gd={sample.EntityCount} cs={mobileObjects.Count}"); + } + + // Battalions the retail sim produced mid-run have no recorded initial + // mapping; pair them with unmapped engine objects in id order so + // counts stay comparable, but exclude them from position parity (the + // engine has no QueueProductionExitUpdate/rally walk yet). + var mappedEngineIds = idMap.Values.ToHashSet(); + var unmappedEngineObjects = mobileObjects.Where(o => !mappedEngineIds.Contains(o.Id)).ToList(); + var unmappedTraceEntities = sample.Entities.Where(e => !idMap.ContainsKey(e.Id)).OrderBy(e => e.Id).ToList(); + for (var i = 0; i < Math.Min(unmappedEngineObjects.Count, unmappedTraceEntities.Count); i++) + { + idMap.Add(unmappedTraceEntities[i].Id, unmappedEngineObjects[i].Id); + producedTraceIds.Add(unmappedTraceEntities[i].Id); + } + + foreach (var entity in sample.Entities) + { + if (!idMap.TryGetValue(entity.Id, out var engineId) || !world.Objects.TryGetValue(engineId, out var engineObject)) + { + continue; + } + var health = engineObject.FindModule()?.Health ?? 0; + if (health != entity.Health) + { + divergences.RecordHealth(sample.Tick, + $" tick {sample.Tick}: entity gd#{entity.Id} health gd={entity.Health} cs={health}"); + } + if (producedTraceIds.Contains(entity.Id)) + { + continue; + } + var dx = ToDouble(engineObject.Position.X) - entity.X; + var dy = ToDouble(engineObject.Position.Y) - entity.Y; + var gap = Math.Sqrt(dx * dx + dy * dy); + divergences.MaxPositionGap = Math.Max(divergences.MaxPositionGap, gap); + if (gap > PositionToleranceUnits) + { + divergences.RecordPosition(sample.Tick, + $" tick {sample.Tick}: entity gd#{entity.Id} position gap {gap:0.###} (gd=({entity.X}, {entity.Y}) cs={engineObject.Position})"); + } + } + } + + private static double ToDouble(Fixed64 value) => (double)value.Raw / Fixed64.OneRaw; + + private static string Describe(int tick) => tick < 0 ? "none" : tick.ToString(); + + private static string? LocateTrace() + { + var overridePath = Environment.GetEnvironmentVariable("OPENBFME_DUALRUN_TRACE"); + if (!string.IsNullOrEmpty(overridePath)) + { + return File.Exists(overridePath) ? overridePath : null; + } + var directory = AppContext.BaseDirectory; + while (directory != null) + { + var candidate = Path.Combine(directory, TraceRelativePath.Replace('/', Path.DirectorySeparatorChar)); + if (File.Exists(candidate)) + { + return candidate; + } + directory = Path.GetDirectoryName(directory); + } + return null; + } + + private sealed class DivergenceReport + { + public int SamplesCompared; + public int ResourceDivergences; + public int CountDivergences; + public int PositionDivergences; + public int HealthDivergences; + public int FirstResourceDivergenceTick = -1; + public int FirstCountDivergenceTick = -1; + public int FirstPositionDivergenceTick = -1; + public int FirstHealthDivergenceTick = -1; + public double MaxPositionGap; + public List Details { get; } = new(); + + public void RecordResources(int tick, string detail) + { + ResourceDivergences++; + if (FirstResourceDivergenceTick < 0) + { + FirstResourceDivergenceTick = tick; + } + Details.Add(detail); + } + + public void RecordCount(int tick, string detail) + { + CountDivergences++; + if (FirstCountDivergenceTick < 0) + { + FirstCountDivergenceTick = tick; + } + Details.Add(detail); + } + + public void RecordPosition(int tick, string detail) + { + PositionDivergences++; + if (FirstPositionDivergenceTick < 0) + { + FirstPositionDivergenceTick = tick; + } + Details.Add(detail); + } + + public void RecordHealth(int tick, string detail) + { + HealthDivergences++; + if (FirstHealthDivergenceTick < 0) + { + FirstHealthDivergenceTick = tick; + } + Details.Add(detail); + } + } +} diff --git a/engine/OpenBfme.Sim.Tests/DualRunTrace.cs b/engine/OpenBfme.Sim.Tests/DualRunTrace.cs new file mode 100644 index 0000000..b202458 --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/DualRunTrace.cs @@ -0,0 +1,236 @@ +using System.Globalization; +using System.Numerics; +using System.Text.Json; + +namespace OpenBfme.Sim.Tests; + +/// +/// Reader for the tier-1 dual-run semantic trace written by the GDScript +/// runner game/tests/retail_dualrun_trace_runner.gd (schema +/// "openbfme.dualrun-trace"). Scalar constants that feed simulation math +/// (speed, tick seconds, coordinates) are read as exact base-10 rationals — +/// raw JSON text through decimal.Parse, never GetDouble — mirroring the +/// PackTemplateLoader float-avoidance strategy, so the mirrored Fixed64 +/// constants are bit-exact. Metrics used only for tolerance reporting +/// (recorded positions) are exposed as double. +/// +public sealed class DualRunTrace +{ + public const string ExpectedSchema = "openbfme.dualrun-trace"; + + public int TotalTicks { get; private init; } + public int SampleInterval { get; private init; } + public long StartingResources { get; private init; } + public long FarmPayoutIntervalTicks { get; private init; } + public long FarmIncomePerPayout { get; private init; } + public string UnitType { get; private init; } = ""; + public long UnitCost { get; private init; } + public long UnitBuildTicks { get; private init; } + public long UnitMemberHealth { get; private init; } + public long UnitMemberCount { get; private init; } + /// Recorded battalion speed in units/second, as an exact rational. + public (long Numerator, long Denominator) SpeedUnitsPerSecond { get; private init; } + /// Recorded seconds per simulation tick, as an exact rational. + public (long Numerator, long Denominator) TickSeconds { get; private init; } + public IReadOnlyList Structures { get; private init; } = Array.Empty(); + public IReadOnlyList InitialEntities { get; private init; } = Array.Empty(); + public IReadOnlyList Commands { get; private init; } = Array.Empty(); + public IReadOnlyList Samples { get; private init; } = Array.Empty(); + + /// Per-tick movement as an exact Fixed64: speed (u/s) x tick seconds (s/tick). + public Fixed64 SpeedPerTick() + { + var numerator = checked(SpeedUnitsPerSecond.Numerator * TickSeconds.Numerator); + var denominator = checked(SpeedUnitsPerSecond.Denominator * TickSeconds.Denominator); + return Fixed64.FromFraction(numerator, denominator); + } + + public static DualRunTrace Load(string path) + { + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var root = document.RootElement; + if (root.GetProperty("schema").GetString() != ExpectedSchema) + { + throw new InvalidDataException($"trace schema is not '{ExpectedSchema}'"); + } + var constants = root.GetProperty("constants"); + var unit = constants.GetProperty("unit"); + + var structures = new List(); + foreach (var row in constants.GetProperty("structures").EnumerateArray()) + { + structures.Add(new TraceStructure( + ReadLong(row.GetProperty("id")), + (int)ReadLong(row.GetProperty("team")), + row.GetProperty("structure_kind").GetString()!, + ReadFixed(row.GetProperty("x")), + ReadFixed(row.GetProperty("y")), + ReadLong(row.GetProperty("maximum_health")), + ReadLong(row.GetProperty("income_per_payout")))); + } + + var initialEntities = new List(); + foreach (var row in root.GetProperty("initial").GetProperty("entities").EnumerateArray()) + { + initialEntities.Add(new TraceEntity( + ReadLong(row.GetProperty("id")), + (int)ReadLong(row.GetProperty("team")), + ReadFixed(row.GetProperty("x")), + ReadFixed(row.GetProperty("y")), + ReadLong(row.GetProperty("health")))); + } + + var commands = new List(); + foreach (var row in root.GetProperty("commands").EnumerateArray()) + { + var args = row.GetProperty("args"); + var type = row.GetProperty("type").GetString()!; + var command = new TraceCommand( + (int)ReadLong(row.GetProperty("tick")), + (int)ReadLong(row.GetProperty("team")), + (int)ReadLong(row.GetProperty("seq")), + type, + row.TryGetProperty("out_of_band", out var oob) && oob.GetBoolean()); + switch (type) + { + case "issue_move": + command = command with + { + TargetIds = args.GetProperty("ids").EnumerateArray().Select(ReadLong).ToArray(), + DestinationX = ReadFixed(args.GetProperty("destination").GetProperty("x")), + DestinationY = ReadFixed(args.GetProperty("destination").GetProperty("y")), + }; + break; + case "queue_unit": + command = command with + { + ProducerId = ReadLong(args.GetProperty("producer")), + UnitType = args.GetProperty("unit_type").GetString(), + }; + break; + case "grant_resources": + command = command with { Amount = ReadLong(args.GetProperty("amount")) }; + break; + default: + throw new InvalidDataException($"trace command type '{type}' has no replay mapping"); + } + commands.Add(command); + } + + var samples = new List(); + foreach (var row in root.GetProperty("samples").EnumerateArray()) + { + var resources = new SortedDictionary(); + foreach (var teamProperty in row.GetProperty("team_resources").EnumerateObject()) + { + resources.Add(int.Parse(teamProperty.Name, CultureInfo.InvariantCulture), ReadLong(teamProperty.Value)); + } + var entities = new List(); + foreach (var entity in row.GetProperty("entities").EnumerateArray()) + { + entities.Add(new TraceSampleEntity( + ReadLong(entity.GetProperty("id")), + (int)ReadLong(entity.GetProperty("team")), + entity.GetProperty("x").GetDouble(), + entity.GetProperty("y").GetDouble(), + ReadLong(entity.GetProperty("health")))); + } + samples.Add(new TraceSample( + (int)ReadLong(row.GetProperty("tick")), + resources, + (int)ReadLong(row.GetProperty("entity_count")), + entities)); + } + + return new DualRunTrace + { + TotalTicks = (int)ReadLong(root.GetProperty("total_ticks")), + SampleInterval = (int)ReadLong(root.GetProperty("sample_interval")), + StartingResources = ReadLong(constants.GetProperty("starting_resources")), + FarmPayoutIntervalTicks = ReadLong(constants.GetProperty("farm_payout_interval_ticks")), + FarmIncomePerPayout = ReadLong(constants.GetProperty("farm_income_per_payout")), + UnitType = unit.GetProperty("unit_type").GetString()!, + UnitCost = ReadLong(unit.GetProperty("cost")), + UnitBuildTicks = ReadLong(unit.GetProperty("build_ticks")), + UnitMemberHealth = ReadLong(unit.GetProperty("member_health")), + UnitMemberCount = ReadLong(unit.GetProperty("member_count")), + SpeedUnitsPerSecond = ReadFraction(unit.GetProperty("speed_units_per_second")), + TickSeconds = ReadFraction(constants.GetProperty("tick_seconds")), + Structures = structures, + InitialEntities = initialEntities, + Commands = commands, + Samples = samples, + }; + } + + private static long ReadLong(JsonElement element) + { + var (numerator, denominator) = ReadFraction(element); + if (denominator != 1) + { + throw new InvalidDataException($"expected an integer, found {element.GetRawText()}"); + } + return numerator; + } + + private static Fixed64 ReadFixed(JsonElement element) + { + var (numerator, denominator) = ReadFraction(element); + return Fixed64.FromFraction(numerator, denominator); + } + + /// + /// Exact rational read of a JSON number: raw text -> decimal (exact + /// base-10) -> reduced integer fraction. Same discipline as + /// PackTemplateLoader.TryReadFraction; no float/double on this path. + /// + private static (long Numerator, long Denominator) ReadFraction(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Number) + { + throw new InvalidDataException($"expected a JSON number, found {element.ValueKind}"); + } + var value = decimal.Parse(element.GetRawText(), NumberStyles.Float, CultureInfo.InvariantCulture); + var bits = decimal.GetBits(value); + var scale = (bits[3] >> 16) & 0xFF; + var negative = (bits[3] & unchecked((int)0x80000000)) != 0; + var magnitude = ((BigInteger)(uint)bits[2] << 64) + | ((BigInteger)(uint)bits[1] << 32) + | (uint)bits[0]; + var num = negative ? -magnitude : magnitude; + var den = BigInteger.Pow(10, scale); + if (!num.IsZero) + { + var gcd = BigInteger.GreatestCommonDivisor(BigInteger.Abs(num), den); + num /= gcd; + den /= gcd; + } + else + { + den = BigInteger.One; + } + if (num > long.MaxValue || num < long.MinValue || den > long.MaxValue) + { + throw new InvalidDataException($"JSON number {element.GetRawText()} does not fit the long rational"); + } + return ((long)num, (long)den); + } +} + +public sealed record TraceStructure(long Id, int Team, string Kind, Fixed64 X, Fixed64 Y, long MaximumHealth, long IncomePerPayout); + +public sealed record TraceEntity(long Id, int Team, Fixed64 X, Fixed64 Y, long Health); + +public sealed record TraceCommand(int Tick, int Team, int Seq, string Type, bool OutOfBand) +{ + public IReadOnlyList? TargetIds { get; init; } + public Fixed64 DestinationX { get; init; } + public Fixed64 DestinationY { get; init; } + public long ProducerId { get; init; } + public string? UnitType { get; init; } + public long Amount { get; init; } +} + +public sealed record TraceSample(int Tick, IReadOnlyDictionary TeamResources, int EntityCount, IReadOnlyList Entities); + +public sealed record TraceSampleEntity(long Id, int Team, double X, double Y, long Health); diff --git a/engine/OpenBfme.Sim.Tests/ModuleBehaviorTests.cs b/engine/OpenBfme.Sim.Tests/ModuleBehaviorTests.cs new file mode 100644 index 0000000..fe103b0 --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/ModuleBehaviorTests.cs @@ -0,0 +1,226 @@ +using OpenBfme.Sim; +using Xunit; + +namespace OpenBfme.Sim.Tests; + +/// Regression tests for the adversarial-review findings plus P1 module semantics. +public class ReviewRegressionTests +{ + [Fact(Timeout = 20000)] + public async Task SqrtTerminatesAndFloorsForAdversarialInputs() + { + // The naive `while (estimate != previous)` Newton loop oscillates between + // k and k+1 for some targets; the monotone form must terminate everywhere + // and return the floor square root. Timeout guards against regressions. + await Task.Run(() => + { + for (long raw = 0; raw <= 5000; raw++) + { + AssertFloorSqrt(raw); + } + var random = new DeterministicRandom(7); + for (var i = 0; i < 5000; i++) + { + var raw = (long)random.NextUInt32() << 31 | random.NextUInt32(); + AssertFloorSqrt(raw & long.MaxValue); + } + // Perfect squares and their neighbours are the classic oscillation bait. + for (long k = 1; k <= 3000; k++) + { + AssertFloorSqrt(k * k - 1); + AssertFloorSqrt(k * k); + AssertFloorSqrt(k * k + 1); + } + }); + } + + private static void AssertFloorSqrt(long raw) + { + var root = Fixed64.Sqrt(Fixed64.FromRaw(raw)); + var target = (System.Numerics.BigInteger)raw << Fixed64.FractionBits; + var square = (System.Numerics.BigInteger)root.Raw * root.Raw; + var nextSquare = ((System.Numerics.BigInteger)root.Raw + 1) * ((System.Numerics.BigInteger)root.Raw + 1); + Assert.True(square <= target, $"sqrt overshoots at raw {raw}"); + Assert.True(nextSquare > target, $"sqrt not tight at raw {raw}"); + } + + [Fact] + public void DeadObjectsDoNotUpdateOnTheirDeathTick() + { + var world = new SimWorld(TestWorlds.Config(), ModuleRegistry.CreateDefault()); + var farm = world.SpawnObject("farm", 0, FixedVector2.Zero); + // Advance to one tick before a payout, then kill via command applied at + // tick start: the farm must NOT collect its aligned payout that tick. + world.Advance(14); + Assert.Equal(0, world.TeamResources(0)); + world.SubmitCommand(TestWorlds.Command(15, 0, 0, "damage", + ("id", CommandValue.OfLong(farm.Id)), ("amount", CommandValue.OfLong(500)))); + world.Advance(1); + Assert.Empty(world.Objects); + Assert.Equal(0, world.TeamResources(0)); + } + + [Fact] + public void SpawningDuringModuleUpdateDoesNotThrowAndStaysDeterministic() + { + SimWorld Build() + { + var world = new SimWorld(BarracksConfig(), ModuleRegistry.CreateDefault()); + var barracks = world.SpawnObject("barracks", 0, FixedVector2.Zero); + barracks.FindModule()!.TryQueue(world, barracks, "soldier"); + barracks.FindModule()!.TryQueue(world, barracks, "soldier"); + return world; + } + + var a = Build(); + var b = Build(); + for (var tick = 1; tick <= 60; tick++) + { + a.Tick(); + b.Tick(); + Assert.Equal(a.StateHash(), b.StateHash()); + } + Assert.Equal(3, a.Objects.Count); + } + + public static SimConfig BarracksConfig() => new( + new[] + { + new ObjectTemplate("soldier", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 120 }), + new ModuleSpec(LinearMoverModule.TypeName, null), + }), + new ObjectTemplate("barracks", new[] + { + new ModuleSpec(GettingBuiltModule.TypeName, new Dictionary { ["ConstructionTicks"] = 10 }), + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 800 }), + new ModuleSpec(ProductionModule.TypeName, new Dictionary + { + ["Build:soldier"] = 20, + }), + }), + new ObjectTemplate("dying_farm", new[] + { + new ModuleSpec(ActiveBodyModule.TypeName, new Dictionary { ["MaxHealth"] = 100 }), + new ModuleSpec(SlowDeathModule.TypeName, new Dictionary { ["DeathTicks"] = 25 }), + new ModuleSpec(ResourceGeneratorModule.TypeName, new Dictionary + { + ["IntervalTicks"] = 10, + ["Amount"] = 7, + }), + }), + }, + randomSeed: 99, + teamCount: 2); +} + +public class P1ModuleTests +{ + [Fact] + public void ProductionWaitsForConstructionThenSpawnsOnSchedule() + { + var world = new SimWorld(ReviewRegressionTests.BarracksConfig(), ModuleRegistry.CreateDefault()); + var barracks = world.SpawnObject("barracks", 0, new FixedVector2(Fixed64.FromInt(5), Fixed64.FromInt(5))); + var production = barracks.FindModule()!; + Assert.True(production.TryQueue(world, barracks, "soldier")); + + // Construction runs ticks 1-10 and clears at the END of tick 10; the + // production module (later in template order) sees the cleared flag the + // same tick, so build ticks are 10..29 and the soldier spawns during + // tick 30 — retail spawn phase: completion tick + 1 (dual-run finding). + world.Advance(9); + Assert.True(barracks.IsUnderConstruction); + Assert.Single(world.Objects); + world.Advance(1); + Assert.False(barracks.IsUnderConstruction); + + world.Advance(19); + Assert.Single(world.Objects); + world.Advance(1); + Assert.Equal(2, world.Objects.Count); + var soldier = world.Objects.Values.First(o => o.TemplateName == "soldier"); + Assert.Equal(new FixedVector2(Fixed64.FromInt(7), Fixed64.FromInt(5)), soldier.Position); + } + + [Fact] + public void ProductionQueueRejectsUnknownTemplatesAndOverflow() + { + var world = new SimWorld(ReviewRegressionTests.BarracksConfig(), ModuleRegistry.CreateDefault()); + var barracks = world.SpawnObject("barracks", 0, FixedVector2.Zero); + var production = barracks.FindModule()!; + Assert.False(production.TryQueue(world, barracks, "catapult")); + for (var i = 0; i < ProductionModule.MaxQueueLength; i++) + { + Assert.True(production.TryQueue(world, barracks, "soldier")); + } + Assert.False(production.TryQueue(world, barracks, "soldier")); + } + + [Fact] + public void SlowDeathDelaysRemovalAndStopsIncome() + { + var world = new SimWorld(ReviewRegressionTests.BarracksConfig(), ModuleRegistry.CreateDefault()); + var farm = world.SpawnObject("dying_farm", 0, FixedVector2.Zero); + world.Advance(10); + Assert.Equal(7, world.TeamResources(0)); + + world.DealDamage(farm, 100); + Assert.Single(world.Objects); + + // Dying object persists for DeathTicks=25 but produces nothing. + world.Advance(24); + Assert.Single(world.Objects); + Assert.Equal(7, world.TeamResources(0)); + world.Advance(1); + Assert.Empty(world.Objects); + } + + [Fact] + public void SlowDeathWorldSnapshotRoundTripsMidDeath() + { + var config = ReviewRegressionTests.BarracksConfig(); + var world = new SimWorld(config, ModuleRegistry.CreateDefault()); + var farm = world.SpawnObject("dying_farm", 0, FixedVector2.Zero); + world.DealDamage(farm, 100); + world.Advance(10); + + var restored = SimWorld.Restore(world.Snapshot(), config, ModuleRegistry.CreateDefault()); + for (var tick = 0; tick < 30; tick++) + { + world.Tick(); + restored.Tick(); + Assert.Equal(world.StateHash(), restored.StateHash()); + } + Assert.Empty(world.Objects); + Assert.Empty(restored.Objects); + } + + [Fact] + public void QueueProductionCommandRoutesThroughTeamValidation() + { + var world = new SimWorld(ReviewRegressionTests.BarracksConfig(), ModuleRegistry.CreateDefault()); + var barracks = world.SpawnObject("barracks", 0, FixedVector2.Zero); + world.Advance(11); // finish construction + + // Wrong team may not queue on this producer. + world.SubmitCommand(TestWorlds.Command(12, 1, 0, "queue_production", + ("id", CommandValue.OfLong(barracks.Id)), ("template", CommandValue.OfString("soldier")))); + // Right team queues successfully. + world.SubmitCommand(TestWorlds.Command(12, 0, 0, "queue_production", + ("id", CommandValue.OfLong(barracks.Id)), ("template", CommandValue.OfString("soldier")))); + world.Advance(1); + Assert.Equal(1, barracks.FindModule()!.QueueLength); + } + + [Fact] + public void DyingObjectsIgnoreFurtherDamage() + { + var world = new SimWorld(ReviewRegressionTests.BarracksConfig(), ModuleRegistry.CreateDefault()); + var farm = world.SpawnObject("dying_farm", 0, FixedVector2.Zero); + world.DealDamage(farm, 100); + var hashAfterDeath = world.StateHash(); + world.DealDamage(farm, 100); + Assert.Equal(hashAfterDeath, world.StateHash()); + } +} diff --git a/engine/OpenBfme.Sim.Tests/OpenBfme.Sim.Tests.csproj b/engine/OpenBfme.Sim.Tests/OpenBfme.Sim.Tests.csproj new file mode 100644 index 0000000..ab8b56e --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/OpenBfme.Sim.Tests.csproj @@ -0,0 +1,21 @@ + + + + net8.0 + enable + enable + false + true + + + + + + + + + + + + + diff --git a/engine/OpenBfme.Sim.Tests/PackTemplateLoaderTests.cs b/engine/OpenBfme.Sim.Tests/PackTemplateLoaderTests.cs new file mode 100644 index 0000000..d8c2b4f --- /dev/null +++ b/engine/OpenBfme.Sim.Tests/PackTemplateLoaderTests.cs @@ -0,0 +1,230 @@ +using System.Text.Json; +using OpenBfme.Sim; +using Xunit; + +namespace OpenBfme.Sim.Tests; + +public class PackTemplateLoaderTests +{ + private static string Document(params string[] rows) => + "{\"schema\":\"openbfme.objects\",\"schemaVersion\":0,\"objects\":[" + string.Join(",", rows) + "]}"; + + private const string UnitRow = """ + { + "id": "test.object.fighter", + "kind": "member", + "displayName": "Fighter", + "simulation": { "health": 200, "speed": 55 } + } + """; + + private const string StructureRow = """ + { + "id": "test.object.barracks", + "kind": "structure", + "displayName": "Barracks", + "simulation": { "health": 3000 } + } + """; + + private const string BattalionRow = """ + { + "id": "test.object.fighter-horde", + "kind": "battalion", + "displayName": "Fighter Horde", + "memberCount": 15, + "commandPoints": 60, + "memberObjectId": "test.object.fighter" + } + """; + + [Fact] + public void UnitRowMapsToActiveBodyAndMover() + { + var result = PackTemplateLoader.LoadFromObjectsDocument(Document(UnitRow)); + + var template = Assert.Single(result.Templates); + Assert.Equal("test.object.fighter", template.Name); + Assert.Equal(2, template.Modules.Count); + + var body = template.Modules[0]; + Assert.Equal(ActiveBodyModule.TypeName, body.TypeName); + Assert.Equal(200, body.GetLong("MaxHealth", -1)); + Assert.Equal("Fighter", body.GetString("DisplayName", "")); + + var mover = template.Modules[1]; + Assert.Equal(LinearMoverModule.TypeName, mover.TypeName); + Assert.Equal(Fixed64.FromFraction(55, SimWorld.TicksPerSecond).Raw, mover.GetLong("SpeedPerTickRaw", -1)); + + Assert.Empty(result.Report.SkippedRows); + } + + [Fact] + public void StructureRowMapsToStructureBodyWithoutMover() + { + var result = PackTemplateLoader.LoadFromObjectsDocument(Document(StructureRow)); + + var template = Assert.Single(result.Templates); + var body = Assert.Single(template.Modules); + Assert.Equal(StructureBodyModule.TypeName, body.TypeName); + Assert.Equal(3000, body.GetLong("MaxHealth", -1)); + } + + [Fact] + public void BattalionRowMapsMemberCountAndCommandPoints() + { + var result = PackTemplateLoader.LoadFromObjectsDocument(Document(BattalionRow)); + + var template = Assert.Single(result.Templates); + var body = Assert.Single(template.Modules); + Assert.Equal(ActiveBodyModule.TypeName, body.TypeName); + Assert.Equal(15, body.GetLong("MemberCount", -1)); + Assert.Equal(60, body.GetLong("CommandPoints", -1)); + Assert.Equal("test.object.fighter", body.GetString("MemberObjectId", "")); + // No health field -> module default, spelled out in the report. + Assert.Contains(result.Report.Notes, note => note.StartsWith("test.object.fighter-horde:", StringComparison.Ordinal)); + } + + [Fact] + public void FractionalSpeedIsExactRationalNotFloat() + { + var row = """{ "id": "test.object.creep", "kind": "member", "simulation": { "health": 10, "speed": 1.15 } }"""; + var result = PackTemplateLoader.LoadFromObjectsDocument(Document(row)); + + var template = Assert.Single(result.Templates); + var mover = template.Modules[1]; + // 1.15 units/second == 115/100 -> per tick 115 / (100 * 30) == 23/600, exactly. + Assert.Equal(Fixed64.FromFraction(23, 600).Raw, mover.GetLong("SpeedPerTickRaw", 0)); + } + + [Fact] + public void UnknownFieldsAreEnumeratedNeverSilent() + { + var row = """ + { + "id": "test.object.odd", + "kind": "member", + "presentation": { "model": "x.glb" }, + "someFutureField": true, + "simulation": { "health": 10, "vision": 175, "cost": 200 } + } + """; + var result = PackTemplateLoader.LoadFromObjectsDocument(Document(row)); + + Assert.Single(result.Templates); + var unmapped = result.Report.UnmappedFields; + Assert.Equal(1, unmapped["presentation"]); + Assert.Equal(1, unmapped["someFutureField"]); + Assert.Equal(1, unmapped["simulation.vision"]); + Assert.Equal(1, unmapped["simulation.cost"]); + } + + [Fact] + public void MalformedRowsAreSkippedWithTypedReasons() + { + var result = PackTemplateLoader.LoadFromObjectsDocument(Document( + "42", // not an object + """{ "kind": "member" }""", // no id + """{ "id": "test.object.nokind" }""", // no kind + """{ "id": "test.object.weird", "kind": "hologram" }""", // unknown kind + """{ "id": "test.object.badhp", "kind": "member", "simulation": { "health": 1.5 } }""", + UnitRow, + UnitRow)); // duplicate id + + Assert.Single(result.Templates); + Assert.Equal(6, result.Report.SkippedRows.Count); + Assert.Equal(RowSkipReason.NotAnObject, result.Report.SkippedRows[0].Reason); + Assert.Equal(RowSkipReason.MissingId, result.Report.SkippedRows[1].Reason); + Assert.Equal(RowSkipReason.MissingKind, result.Report.SkippedRows[2].Reason); + Assert.Equal(RowSkipReason.UnknownKind, result.Report.SkippedRows[3].Reason); + Assert.Equal(RowSkipReason.InvalidNumericField, result.Report.SkippedRows[4].Reason); + Assert.Equal(RowSkipReason.DuplicateId, result.Report.SkippedRows[5].Reason); + Assert.Equal(6, result.Report.SkippedRows[5].Index); + } + + [Fact] + public void UnusableDocumentThrowsTypedError() + { + Assert.Throws(() => PackTemplateLoader.LoadFromObjectsDocument("not json")); + Assert.Throws(() => PackTemplateLoader.LoadFromObjectsDocument("[]")); + Assert.Throws(() => PackTemplateLoader.LoadFromObjectsDocument("{\"objects\":7}")); + Assert.Throws( + () => PackTemplateLoader.LoadFromObjectsDocument("{\"schema\":\"wrong.schema\",\"objects\":[]}")); + } + + [Fact] + public void LoadingTwiceYieldsHashIdenticalTwinRunsOver500Ticks() + { + var json = Document(UnitRow, StructureRow, BattalionRow); + var hashA = RunWorld(PackTemplateLoader.LoadFromObjectsDocument(json).Templates, ticks: 500); + var hashB = RunWorld(PackTemplateLoader.LoadFromObjectsDocument(json).Templates, ticks: 500); + Assert.Equal(hashA, hashB); + } + + [Fact] + public void RealPackObjectsLoadSpawnAndStayDeterministicOver300Ticks() + { + var objectsJsonPath = FindActivePackObjectsJson(); + if (objectsJsonPath == null) + { + return; // pack bundle absent on this machine — integration coverage skipped + } + var json = File.ReadAllText(objectsJsonPath); + var result = PackTemplateLoader.LoadFromObjectsDocument(json); + + Assert.True(result.Templates.Count > 0, "real pack produced no templates"); + Assert.Empty(result.Report.SkippedRows); + + var hashA = RunWorld(result.Templates, ticks: 300); + var hashB = RunWorld(PackTemplateLoader.LoadFromObjectsDocument(json).Templates, ticks: 300); + Assert.Equal(hashA, hashB); + } + + /// Spawns one of each template, nudges every mover, advances, returns the state hash. + private static string RunWorld(IReadOnlyList templates, int ticks) + { + var config = new SimConfig(templates, randomSeed: 2026, teamCount: 2); + var world = new SimWorld(config, ModuleRegistry.CreateDefault()); + var slot = 0; + foreach (var template in templates) + { + var spawned = world.SpawnObject(template.Name, slot % 2, + new FixedVector2(Fixed64.FromInt(slot * 10), Fixed64.FromInt(slot * 7))); + spawned.FindModule()?.SetDestination( + new FixedVector2(Fixed64.FromInt(slot * 10 + 900), Fixed64.FromInt(slot * 7 + 400))); + slot++; + } + world.Advance(ticks); + return world.StateHash(); + } + + /// + /// Walks up from the test bin dir to the repo root, reads + /// .private/content-packs/selection.json, and resolves the active pack's + /// data/objects.json. Returns null (test skips) when anything is absent. + /// + private static string? FindActivePackObjectsJson() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null) + { + var selectionPath = Path.Combine(dir.FullName, ".private", "content-packs", "selection.json"); + if (File.Exists(selectionPath)) + { + using var selection = JsonDocument.Parse(File.ReadAllText(selectionPath)); + if (!selection.RootElement.TryGetProperty("activePack", out var activePack) + || activePack.ValueKind != JsonValueKind.String) + { + return null; + } + var objectsPath = Path.Combine( + dir.FullName, ".private", "content-packs", + activePack.GetString()!.Replace('/', Path.DirectorySeparatorChar), + "data", "objects.json"); + return File.Exists(objectsPath) ? objectsPath : null; + } + dir = dir.Parent; + } + return null; + } +} diff --git a/engine/OpenBfme.Sim/CanonicalState.cs b/engine/OpenBfme.Sim/CanonicalState.cs new file mode 100644 index 0000000..a816f69 --- /dev/null +++ b/engine/OpenBfme.Sim/CanonicalState.cs @@ -0,0 +1,87 @@ +using System.Security.Cryptography; +using System.Text; + +namespace OpenBfme.Sim; + +/// +/// Canonical little-endian binary writer. Every piece of authoritative state is +/// serialized through this type, so the state hash and the snapshot format are +/// the same bytes by construction. +/// +public sealed class CanonicalWriter +{ + private readonly MemoryStream _stream = new(); + private readonly BinaryWriter _writer; + + public CanonicalWriter() + { + _writer = new BinaryWriter(_stream, Encoding.UTF8, leaveOpen: true); + } + + public void WriteByte(byte value) => _writer.Write(value); + public void WriteInt(int value) => _writer.Write(value); + public void WriteLong(long value) => _writer.Write(value); + public void WriteBool(bool value) => _writer.Write(value ? (byte)1 : (byte)0); + public void WriteFixed(Fixed64 value) => _writer.Write(value.Raw); + + public void WriteVector(FixedVector2 value) + { + _writer.Write(value.X.Raw); + _writer.Write(value.Y.Raw); + } + + public void WriteString(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + _writer.Write(bytes.Length); + _writer.Write(bytes); + } + + public byte[] ToArray() + { + _writer.Flush(); + return _stream.ToArray(); + } + + public string ToSha256Hex() + { + _writer.Flush(); + return Convert.ToHexString(SHA256.HashData(_stream.ToArray())).ToLowerInvariant(); + } +} + +public sealed class CanonicalReader +{ + private readonly BinaryReader _reader; + + public CanonicalReader(byte[] payload) + { + _reader = new BinaryReader(new MemoryStream(payload, writable: false), Encoding.UTF8); + } + + public byte ReadByte() => _reader.ReadByte(); + public int ReadInt() => _reader.ReadInt32(); + public long ReadLong() => _reader.ReadInt64(); + public bool ReadBool() => _reader.ReadByte() != 0; + public Fixed64 ReadFixed() => Fixed64.FromRaw(_reader.ReadInt64()); + public FixedVector2 ReadVector() => new(Fixed64.FromRaw(_reader.ReadInt64()), Fixed64.FromRaw(_reader.ReadInt64())); + + public string ReadString() + { + var length = _reader.ReadInt32(); + if (length < 0) + { + throw new InvalidDataException("Negative string length in canonical payload"); + } + return Encoding.UTF8.GetString(_reader.ReadBytes(length)); + } + + public void ExpectEnd() + { + if (_reader.BaseStream.Position != _reader.BaseStream.Length) + { + throw new InvalidDataException( + $"Canonical payload has {_reader.BaseStream.Length - _reader.BaseStream.Position} trailing bytes"); + } + } +} diff --git a/engine/OpenBfme.Sim/CombatModules.cs b/engine/OpenBfme.Sim/CombatModules.cs new file mode 100644 index 0000000..e98da78 --- /dev/null +++ b/engine/OpenBfme.Sim/CombatModules.cs @@ -0,0 +1,196 @@ +namespace OpenBfme.Sim; + +/// Canonical damage-type identifiers. Basis for armor scaling and crush gating. +public static class DamageTypes +{ + public const string Default = "default"; + public const string Slash = "slash"; + public const string Pierce = "pierce"; + public const string Siege = "siege"; + public const string Crush = "crush"; +} + +/// +/// ArmorSet-shaped scaling (SAGE armor.ini semantics): per-damage-type basis +/// points, 10000 = 100% damage taken. Design data keys: "Armor:<type>"; +/// unlisted types take "ArmorDefault" (default 10000). +/// +public sealed class ArmorModule : ModuleBase +{ + public const string TypeName = "Armor"; + + public ArmorModule(ModuleSpec spec) : base(spec) + { + } + + public override long ModifyIncomingDamage(GameObject self, string damageType, long amount) + { + var basisPoints = Spec.GetLong("Armor:" + damageType, Spec.GetLong("ArmorDefault", 10_000)); + basisPoints = Math.Clamp(basisPoints, 0, 100_000); + return amount * basisPoints / 10_000; + } +} + +/// +/// SquishCollide marker (374 objects): declares the object crushable. Crush +/// damage is dropped by the world for objects without this module. +/// +public sealed class SquishCollideModule : ModuleBase +{ + public const string TypeName = "SquishCollide"; + + public SquishCollideModule(ModuleSpec spec) : base(spec) + { + } +} + +/// +/// Single-weapon attack cycle: fires at the current target every ReloadTicks +/// when within RangeRaw, dealing Damage of DamageType. Target selection is the +/// AI module's job; this module owns only the firing mechanics. +/// +public sealed class WeaponModule : ModuleBase +{ + public const string TypeName = "Weapon"; + + private readonly Fixed64 _range; + private readonly long _damage; + private readonly int _reloadTicks; + private readonly string _damageType; + private int _cooldown; + private int _targetId; + + public WeaponModule(ModuleSpec spec) : base(spec) + { + _range = spec.GetFixed("RangeRaw", Fixed64.FromInt(2)); + _damage = spec.GetLong("Damage", 10); + _reloadTicks = (int)Math.Max(1, spec.GetLong("ReloadTicks", 10)); + _damageType = spec.GetString("DamageType", DamageTypes.Default); + } + + public int TargetId => _targetId; + public Fixed64 Range => _range; + + public void SetTarget(int targetId) => _targetId = targetId; + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (_cooldown > 0) + { + _cooldown--; + } + if (self.IsUnderConstruction || self.IsDying || _targetId == 0) + { + return; + } + if (!world.Objects.TryGetValue(_targetId, out var target) || target.IsDead || target.IsDying) + { + _targetId = 0; + return; + } + var rangeSquared = _range * _range; + if (self.Position.DistanceSquaredTo(target.Position) > rangeSquared) + { + return; + } + if (_cooldown > 0) + { + return; + } + world.DealDamage(target, _damage, _damageType); + _cooldown = _reloadTicks; + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteInt(_cooldown); + writer.WriteInt(_targetId); + } + + public override void ReadState(CanonicalReader reader) + { + _cooldown = reader.ReadInt(); + _targetId = reader.ReadInt(); + } +} + +/// +/// AIUpdateInterface-lite (539 objects): acquires the nearest living enemy +/// within VisionRangeRaw (ties broken by lowest id — deterministic), hands it +/// to the weapon, and walks into weapon range via the LinearMover when needed. +/// +public sealed class AiCombatModule : ModuleBase +{ + public const string TypeName = "AiCombat"; + + private readonly Fixed64 _visionRange; + private readonly int _scanIntervalTicks; + private int _ticksUntilScan; + + public AiCombatModule(ModuleSpec spec) : base(spec) + { + _visionRange = spec.GetFixed("VisionRangeRaw", Fixed64.FromInt(12)); + _scanIntervalTicks = (int)Math.Max(1, spec.GetLong("ScanIntervalTicks", 5)); + _ticksUntilScan = 1; + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (self.IsUnderConstruction || self.IsDying) + { + return; + } + var weapon = self.FindModule(); + if (weapon == null) + { + return; + } + _ticksUntilScan--; + if (_ticksUntilScan <= 0) + { + _ticksUntilScan = _scanIntervalTicks; + if (weapon.TargetId == 0) + { + weapon.SetTarget(FindNearestEnemyId(world, self)); + } + } + if (weapon.TargetId == 0 || !world.Objects.TryGetValue(weapon.TargetId, out var target)) + { + return; + } + var mover = self.FindModule(); + if (mover == null) + { + return; + } + var rangeSquared = weapon.Range * weapon.Range; + if (self.Position.DistanceSquaredTo(target.Position) > rangeSquared) + { + mover.SetDestination(target.Position); + } + } + + private int FindNearestEnemyId(SimWorld world, GameObject self) + { + var bestId = 0; + var bestDistanceSquared = _visionRange * _visionRange; + foreach (var candidate in world.Objects.Values) + { + if (candidate.Team == self.Team || candidate.IsDead || candidate.IsDying) + { + continue; + } + var distanceSquared = self.Position.DistanceSquaredTo(candidate.Position); + // Strict < with ascending-id iteration = lowest id wins ties. Deterministic. + if (distanceSquared < bestDistanceSquared || (bestId == 0 && distanceSquared == bestDistanceSquared)) + { + bestId = candidate.Id; + bestDistanceSquared = distanceSquared; + } + } + return bestId; + } + + public override void WriteState(CanonicalWriter writer) => writer.WriteInt(_ticksUntilScan); + public override void ReadState(CanonicalReader reader) => _ticksUntilScan = reader.ReadInt(); +} diff --git a/engine/OpenBfme.Sim/DeterministicRandom.cs b/engine/OpenBfme.Sim/DeterministicRandom.cs new file mode 100644 index 0000000..1282378 --- /dev/null +++ b/engine/OpenBfme.Sim/DeterministicRandom.cs @@ -0,0 +1,62 @@ +namespace OpenBfme.Sim; + +/// +/// PCG32 generator. The stream is part of authoritative state: identical seeds and +/// identical call sequences yield identical values on every platform, and the +/// internal state serializes with the snapshot. +/// +public sealed class DeterministicRandom +{ + private const ulong Multiplier = 6364136223846793005UL; + + private ulong _state; + private readonly ulong _increment; + + public DeterministicRandom(ulong seed, ulong sequence = 0) + { + _increment = (sequence << 1) | 1UL; + _state = 0; + NextUInt32(); + _state += seed; + NextUInt32(); + } + + private DeterministicRandom(ulong state, ulong increment, bool restored) + { + _ = restored; + _state = state; + _increment = increment; + } + + public uint NextUInt32() + { + var oldState = _state; + _state = unchecked(oldState * Multiplier + _increment); + var xorShifted = (uint)(((oldState >> 18) ^ oldState) >> 27); + var rotation = (int)(oldState >> 59); + return (xorShifted >> rotation) | (xorShifted << (-rotation & 31)); + } + + /// Uniform value in [0, exclusiveUpperBound) without modulo bias. + public uint NextBelow(uint exclusiveUpperBound) + { + if (exclusiveUpperBound == 0) + { + throw new ArgumentOutOfRangeException(nameof(exclusiveUpperBound)); + } + var threshold = (uint)(-exclusiveUpperBound) % exclusiveUpperBound; + while (true) + { + var candidate = NextUInt32(); + if (candidate >= threshold) + { + return candidate % exclusiveUpperBound; + } + } + } + + public (ulong State, ulong Increment) Serialize() => (_state, _increment); + + public static DeterministicRandom Deserialize(ulong state, ulong increment) => + new(state, increment, restored: true); +} diff --git a/engine/OpenBfme.Sim/Fixed64.cs b/engine/OpenBfme.Sim/Fixed64.cs new file mode 100644 index 0000000..cca8f00 --- /dev/null +++ b/engine/OpenBfme.Sim/Fixed64.cs @@ -0,0 +1,121 @@ +namespace OpenBfme.Sim; + +/// +/// Q32.32 signed fixed-point number. The only scalar type permitted in simulation +/// math — floats are banned from authoritative state so that every platform +/// produces bit-identical results under lockstep. +/// +public readonly struct Fixed64 : IEquatable, IComparable +{ + public const int FractionBits = 32; + public const long OneRaw = 1L << FractionBits; + + public static readonly Fixed64 Zero = new(0); + public static readonly Fixed64 One = new(OneRaw); + public static readonly Fixed64 Half = new(OneRaw >> 1); + public static readonly Fixed64 MaxValue = new(long.MaxValue); + public static readonly Fixed64 MinValue = new(long.MinValue); + + public long Raw { get; } + + private Fixed64(long raw) => Raw = raw; + + public static Fixed64 FromRaw(long raw) => new(raw); + public static Fixed64 FromInt(int value) => new((long)value << FractionBits); + + /// Exact rational construction; the only sanctioned bridge from design data. + public static Fixed64 FromFraction(long numerator, long denominator) + { + if (denominator == 0) + { + throw new DivideByZeroException("Fixed64.FromFraction denominator is zero"); + } + return new Fixed64((long)(((System.Numerics.BigInteger)numerator << FractionBits) / denominator)); + } + + public int ToIntFloor() => (int)(Raw >> FractionBits); + + public static Fixed64 operator +(Fixed64 a, Fixed64 b) => new(checked(a.Raw + b.Raw)); + public static Fixed64 operator -(Fixed64 a, Fixed64 b) => new(checked(a.Raw - b.Raw)); + public static Fixed64 operator -(Fixed64 a) => new(checked(-a.Raw)); + + public static Fixed64 operator *(Fixed64 a, Fixed64 b) + { + var product = (System.Numerics.BigInteger)a.Raw * b.Raw >> FractionBits; + if (product > long.MaxValue || product < long.MinValue) + { + throw new OverflowException("Fixed64 multiplication overflow"); + } + return new Fixed64((long)product); + } + + public static Fixed64 operator /(Fixed64 a, Fixed64 b) + { + if (b.Raw == 0) + { + throw new DivideByZeroException("Fixed64 division by zero"); + } + var quotient = ((System.Numerics.BigInteger)a.Raw << FractionBits) / b.Raw; + if (quotient > long.MaxValue || quotient < long.MinValue) + { + throw new OverflowException("Fixed64 division overflow"); + } + return new Fixed64((long)quotient); + } + + public static bool operator <(Fixed64 a, Fixed64 b) => a.Raw < b.Raw; + public static bool operator >(Fixed64 a, Fixed64 b) => a.Raw > b.Raw; + public static bool operator <=(Fixed64 a, Fixed64 b) => a.Raw <= b.Raw; + public static bool operator >=(Fixed64 a, Fixed64 b) => a.Raw >= b.Raw; + public static bool operator ==(Fixed64 a, Fixed64 b) => a.Raw == b.Raw; + public static bool operator !=(Fixed64 a, Fixed64 b) => a.Raw != b.Raw; + + public static Fixed64 Min(Fixed64 a, Fixed64 b) => a.Raw <= b.Raw ? a : b; + public static Fixed64 Max(Fixed64 a, Fixed64 b) => a.Raw >= b.Raw ? a : b; + public static Fixed64 Abs(Fixed64 a) => a.Raw < 0 ? new Fixed64(checked(-a.Raw)) : a; + + public static Fixed64 Clamp(Fixed64 value, Fixed64 min, Fixed64 max) + { + if (min > max) + { + throw new ArgumentException("Fixed64.Clamp min exceeds max"); + } + return value < min ? min : value > max ? max : value; + } + + /// Integer Newton-Raphson square root; deterministic on every platform. + public static Fixed64 Sqrt(Fixed64 value) + { + if (value.Raw < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Fixed64.Sqrt of negative value"); + } + if (value.Raw == 0) + { + return Zero; + } + // Monotone integer Newton (x, y=(x+t/x)/2 while y> 1; + while (y < x) + { + x = y; + y = (x + target / x) >> 1; + } + return new Fixed64((long)x); + } + + public bool Equals(Fixed64 other) => Raw == other.Raw; + public override bool Equals(object? obj) => obj is Fixed64 other && Equals(other); + public override int GetHashCode() => Raw.GetHashCode(); + public int CompareTo(Fixed64 other) => Raw.CompareTo(other.Raw); + + public override string ToString() + { + // Diagnostic only — never feeds simulation math. + return ((double)Raw / OneRaw).ToString("0.######", System.Globalization.CultureInfo.InvariantCulture); + } +} diff --git a/engine/OpenBfme.Sim/FixedVector2.cs b/engine/OpenBfme.Sim/FixedVector2.cs new file mode 100644 index 0000000..d06e798 --- /dev/null +++ b/engine/OpenBfme.Sim/FixedVector2.cs @@ -0,0 +1,40 @@ +namespace OpenBfme.Sim; + +/// World-plane vector (X/Z ground coordinates), fixed-point throughout. +public readonly struct FixedVector2 : IEquatable +{ + public static readonly FixedVector2 Zero = new(Fixed64.Zero, Fixed64.Zero); + + public Fixed64 X { get; } + public Fixed64 Y { get; } + + public FixedVector2(Fixed64 x, Fixed64 y) + { + X = x; + Y = y; + } + + public static FixedVector2 operator +(FixedVector2 a, FixedVector2 b) => new(a.X + b.X, a.Y + b.Y); + public static FixedVector2 operator -(FixedVector2 a, FixedVector2 b) => new(a.X - b.X, a.Y - b.Y); + public static FixedVector2 operator *(FixedVector2 a, Fixed64 scalar) => new(a.X * scalar, a.Y * scalar); + + public static bool operator ==(FixedVector2 a, FixedVector2 b) => a.X == b.X && a.Y == b.Y; + public static bool operator !=(FixedVector2 a, FixedVector2 b) => !(a == b); + + public Fixed64 LengthSquared() => X * X + Y * Y; + public Fixed64 Length() => Fixed64.Sqrt(LengthSquared()); + + public Fixed64 DistanceSquaredTo(FixedVector2 other) => (this - other).LengthSquared(); + + /// Zero vector normalizes to zero rather than throwing — callers branch on it. + public FixedVector2 Normalized() + { + var length = Length(); + return length == Fixed64.Zero ? Zero : new FixedVector2(X / length, Y / length); + } + + public bool Equals(FixedVector2 other) => this == other; + public override bool Equals(object? obj) => obj is FixedVector2 other && Equals(other); + public override int GetHashCode() => HashCode.Combine(X.Raw, Y.Raw); + public override string ToString() => $"({X}, {Y})"; +} diff --git a/engine/OpenBfme.Sim/GameObject.cs b/engine/OpenBfme.Sim/GameObject.cs new file mode 100644 index 0000000..a32960f --- /dev/null +++ b/engine/OpenBfme.Sim/GameObject.cs @@ -0,0 +1,58 @@ +namespace OpenBfme.Sim; + +public sealed class GameObject +{ + public int Id { get; } + public string TemplateName { get; } + public int Team { get; } + public FixedVector2 Position { get; private set; } + public bool IsDead { get; private set; } + /// Death claimed by a module (SlowDeath): still in the world, no longer operational. + public bool IsDying { get; private set; } + public bool IsUnderConstruction { get; private set; } + public IReadOnlyList Modules { get; } + + internal GameObject(int id, string templateName, int team, FixedVector2 position, IReadOnlyList modules) + { + Id = id; + TemplateName = templateName; + Team = team; + Position = position; + Modules = modules; + } + + public void SetPosition(FixedVector2 position) => Position = position; + + public void MarkDead() => IsDead = true; + + public void MarkDying() => IsDying = true; + + public void SetUnderConstruction(bool value) => IsUnderConstruction = value; + + public T? FindModule() where T : ModuleBase + { + foreach (var module in Modules) + { + if (module is T typed) + { + return typed; + } + } + return null; + } + + internal void WriteState(CanonicalWriter writer) + { + writer.WriteInt(Id); + writer.WriteString(TemplateName); + writer.WriteInt(Team); + writer.WriteVector(Position); + writer.WriteBool(IsDead); + writer.WriteBool(IsDying); + writer.WriteBool(IsUnderConstruction); + foreach (var module in Modules) + { + module.WriteState(writer); + } + } +} diff --git a/engine/OpenBfme.Sim/GapModules.cs b/engine/OpenBfme.Sim/GapModules.cs new file mode 100644 index 0000000..9b3c80a --- /dev/null +++ b/engine/OpenBfme.Sim/GapModules.cs @@ -0,0 +1,312 @@ +namespace OpenBfme.Sim; + +/// +/// HordeContain-shaped container (122 objects in the union corpus): the horde +/// object holds MemberCount member slots as DATA — this tier spawns no child +/// sim objects (member positions/formations are presentation; child-object +/// containment arrives with the horde AI lane). Aggregate health delegates to +/// the members: incoming damage fills member slots in ascending slot order +/// (lowest living slot first, overflow kills through to the next slot), which +/// is deterministic by construction. When every member is at zero the horde +/// routes through the normal death pipeline. +/// Design data: MemberCount (default 1), MemberHealth per member (default 100). +/// +public sealed class HordeContainModule : ModuleBase +{ + public const string TypeName = "HordeContain"; + + private readonly long _memberMaxHealth; + private readonly long[] _memberHealth; + + public HordeContainModule(ModuleSpec spec) : base(spec) + { + var memberCount = (int)Math.Clamp(spec.GetLong("MemberCount", 1), 1, 1024); + _memberMaxHealth = Math.Max(1, spec.GetLong("MemberHealth", 100)); + _memberHealth = new long[memberCount]; + for (var i = 0; i < _memberHealth.Length; i++) + { + _memberHealth[i] = _memberMaxHealth; + } + } + + public int MemberCount => _memberHealth.Length; + public long MemberMaxHealth => _memberMaxHealth; + public long MemberHealthAt(int slot) => _memberHealth[slot]; + + public int AliveMemberCount + { + get + { + var alive = 0; + foreach (var health in _memberHealth) + { + if (health > 0) + { + alive++; + } + } + return alive; + } + } + + public long TotalHealth + { + get + { + long total = 0; + foreach (var health in _memberHealth) + { + total += health; + } + return total; + } + } + + public override bool OnDamage(SimWorld world, GameObject self, long amount) + { + if (amount < 0) + { + throw new ArgumentOutOfRangeException(nameof(amount), "Damage must be non-negative"); + } + if (TotalHealth == 0) + { + return true; + } + for (var slot = 0; slot < _memberHealth.Length && amount > 0; slot++) + { + if (_memberHealth[slot] == 0) + { + continue; + } + var applied = Math.Min(_memberHealth[slot], amount); + _memberHealth[slot] -= applied; + amount -= applied; + } + if (TotalHealth == 0) + { + world.HandleDeath(self); + } + return true; + } + + public override void WriteState(CanonicalWriter writer) + { + // Member count is config; only the healths are mutable state. + foreach (var health in _memberHealth) + { + writer.WriteLong(health); + } + } + + public override void ReadState(CanonicalReader reader) + { + for (var i = 0; i < _memberHealth.Length; i++) + { + _memberHealth[i] = reader.ReadLong(); + } + } +} + +/// +/// BezierProjectileBehavior-lite (237 objects in the union corpus): the module +/// TYPE maps now; trajectory fidelity is presentation. This tier flies a +/// straight line — Launch aims at a target object, the projectile closes the +/// remaining offset in equal fractions over FlightTicks updates (re-aiming at +/// the target's current position each tick while it lives), and on the arrival +/// update deals Damage of DamageType via world.DealDamage, then expires through +/// the death pipeline. The full bezier arc is a later fidelity pass. +/// Design data: FlightTicks (default 10), Damage (default 10), +/// string DamageType (default "default"). +/// +public sealed class BezierProjectileModule : ModuleBase +{ + public const string TypeName = "BezierProjectile"; + + private readonly int _flightTicks; + private readonly long _damage; + private readonly string _damageType; + + private bool _inFlight; + private int _targetId; + private FixedVector2 _aimPoint; + private int _ticksRemaining; + + public BezierProjectileModule(ModuleSpec spec) : base(spec) + { + _flightTicks = (int)Math.Max(1, spec.GetLong("FlightTicks", 10)); + _damage = Math.Max(0, spec.GetLong("Damage", 10)); + _damageType = spec.GetString("DamageType", DamageTypes.Default); + } + + public bool InFlight => _inFlight; + public int TargetId => _targetId; + + /// + /// Arms the projectile at the target: it arrives (and deals damage) on its + /// FlightTicks-th update after launch. + /// + public void Launch(SimWorld world, GameObject self, int targetId) + { + _inFlight = true; + _targetId = targetId; + _aimPoint = world.Objects.TryGetValue(targetId, out var target) ? target.Position : self.Position; + _ticksRemaining = _flightTicks; + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (!_inFlight || self.IsDying) + { + return; + } + var target = world.Objects.TryGetValue(_targetId, out var found) && !found.IsDead ? found : null; + if (target != null) + { + _aimPoint = target.Position; + } + // Equal-fraction closing: step = remaining offset / remaining ticks. + // The final tick's step is the whole remaining offset — exact arrival. + var offset = _aimPoint - self.Position; + var ticks = Fixed64.FromInt(_ticksRemaining); + var step = new FixedVector2(offset.X / ticks, offset.Y / ticks); + self.SetPosition(_ticksRemaining == 1 ? _aimPoint : self.Position + step); + _ticksRemaining--; + if (_ticksRemaining > 0) + { + return; + } + _inFlight = false; + if (target != null && _damage > 0) + { + world.DealDamage(target, _damage, _damageType); + } + world.HandleDeath(self); + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteBool(_inFlight); + writer.WriteInt(_targetId); + writer.WriteVector(_aimPoint); + writer.WriteInt(_ticksRemaining); + } + + public override void ReadState(CanonicalReader reader) + { + _inFlight = reader.ReadBool(); + _targetId = reader.ReadInt(); + _aimPoint = reader.ReadVector(); + _ticksRemaining = reader.ReadInt(); + } +} + +/// +/// AttributeModifierAuraUpdate-lite (227 objects in the union corpus): a radius +/// aura granting a flat ARMOR basis-point modifier to allied objects in range +/// (the single chosen integration for this tier — the damage-bonus side arrives +/// with the weapon-modifier lane). Every RecomputeTicks the aura rescans +/// world.Objects in ascending id order (deterministic) and caches the allied +/// ids within RadiusRaw; at the END of each world tick, SimWorld rebuilds its +/// aura armor table from every living, non-dying, constructed carrier's cache, +/// so the table is stable for the whole following tick (commands and the module +/// sweep alike). world.DealDamage consults the table after the per-module +/// ModifyIncomingDamage chain. +/// +/// STACKING RULE (documented contract): contributions from multiple aura +/// carriers ADD; the summed basis points are clamped to [0, 10000] at damage +/// application, so stacked auras can reach full immunity but never heal. +/// The table is derived state — rebuilt deterministically from module caches — +/// so it is NOT serialized; SimWorld.Restore rebuilds it after loading. +/// Design data: RadiusRaw (Fixed64 raw, default 10), RecomputeTicks (default 5), +/// ArmorBonusBp (default 0; 1000 = 10% less incoming damage). +/// Aura carriers do not buff themselves. +/// +public sealed class AttributeModifierAuraModule : ModuleBase +{ + public const string TypeName = "AttributeModifierAura"; + + private readonly Fixed64 _radius; + private readonly int _recomputeTicks; + private readonly long _armorBonusBp; + + private int _ticksUntilScan; + private readonly List _affectedIds = new(); + + public AttributeModifierAuraModule(ModuleSpec spec) : base(spec) + { + _radius = spec.GetFixed("RadiusRaw", Fixed64.FromInt(10)); + _recomputeTicks = (int)Math.Max(1, spec.GetLong("RecomputeTicks", 5)); + _armorBonusBp = Math.Clamp(spec.GetLong("ArmorBonusBp", 0), 0, 10_000); + _ticksUntilScan = 1; // first update scans immediately + } + + public long ArmorBonusBp => _armorBonusBp; + public IReadOnlyList AffectedIds => _affectedIds; + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (self.IsUnderConstruction || self.IsDying) + { + return; + } + _ticksUntilScan--; + if (_ticksUntilScan > 0) + { + return; + } + _ticksUntilScan = _recomputeTicks; + _affectedIds.Clear(); + var radiusSquared = _radius * _radius; + foreach (var candidate in world.Objects.Values) // ascending id — deterministic + { + if (candidate.Id == self.Id || candidate.Team != self.Team + || candidate.IsDead || candidate.IsDying) + { + continue; + } + if (self.Position.DistanceSquaredTo(candidate.Position) <= radiusSquared) + { + _affectedIds.Add(candidate.Id); + } + } + } + + /// Adds this aura's cached contributions into the world table (additive stacking). + internal void ContributeTo(SortedDictionary armorBonusBpByObjectId, SimWorld world) + { + if (_armorBonusBp == 0) + { + return; + } + foreach (var id in _affectedIds) + { + if (!world.Objects.ContainsKey(id)) + { + continue; // member died since the last rescan + } + armorBonusBpByObjectId[id] = + armorBonusBpByObjectId.TryGetValue(id, out var existing) ? existing + _armorBonusBp : _armorBonusBp; + } + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteInt(_ticksUntilScan); + writer.WriteInt(_affectedIds.Count); + foreach (var id in _affectedIds) + { + writer.WriteInt(id); + } + } + + public override void ReadState(CanonicalReader reader) + { + _ticksUntilScan = reader.ReadInt(); + _affectedIds.Clear(); + var count = reader.ReadInt(); + for (var i = 0; i < count; i++) + { + _affectedIds.Add(reader.ReadInt()); + } + } +} diff --git a/engine/OpenBfme.Sim/Modules.cs b/engine/OpenBfme.Sim/Modules.cs new file mode 100644 index 0000000..3f7691a --- /dev/null +++ b/engine/OpenBfme.Sim/Modules.cs @@ -0,0 +1,729 @@ +namespace OpenBfme.Sim; + +/// +/// Static per-template module description: the module's SAGE type name plus its +/// design-data dictionary (parsed upstream from INI via the importer's compiled +/// manifests). Data is config, not state — it is never mutated at runtime. +/// +public sealed class ModuleSpec +{ + public string TypeName { get; } + public IReadOnlyDictionary Data => _data; + public IReadOnlyDictionary StringData => _stringData; + + private readonly SortedDictionary _data; + private readonly SortedDictionary _stringData; + + public ModuleSpec( + string typeName, + IEnumerable>? data = null, + IEnumerable>? stringData = null) + { + TypeName = typeName ?? throw new ArgumentNullException(nameof(typeName)); + _data = new SortedDictionary(StringComparer.Ordinal); + if (data != null) + { + foreach (var pair in data) + { + _data.Add(pair.Key, pair.Value); + } + } + _stringData = new SortedDictionary(StringComparer.Ordinal); + if (stringData != null) + { + foreach (var pair in stringData) + { + _stringData.Add(pair.Key, pair.Value); + } + } + } + + public long GetLong(string key, long fallback) => _data.TryGetValue(key, out var value) ? value : fallback; + + public Fixed64 GetFixed(string key, Fixed64 fallback) => + _data.TryGetValue(key, out var value) ? Fixed64.FromRaw(value) : fallback; + + public string GetString(string key, string fallback) => + _stringData.TryGetValue(key, out var value) ? value : fallback; +} + +public sealed class ObjectTemplate +{ + public string Name { get; } + public IReadOnlyList Modules { get; } + + public ObjectTemplate(string name, IReadOnlyList modules) + { + Name = name ?? throw new ArgumentNullException(nameof(name)); + Modules = modules ?? throw new ArgumentNullException(nameof(modules)); + } +} + +/// +/// Base class for simulation modules. Update order is the template's module order, +/// applied to objects in ascending id order — both fixed, both part of the +/// determinism contract. Mutable module state must round-trip through +/// WriteState/ReadState or the snapshot/hash gates will catch it. +/// +public abstract class ModuleBase +{ + protected ModuleBase(ModuleSpec spec) => Spec = spec; + + public ModuleSpec Spec { get; } + + public virtual void OnUpdate(SimWorld world, GameObject self) + { + } + + /// Returns true when the damage was consumed (e.g. by a body module). + public virtual bool OnDamage(SimWorld world, GameObject self, long amount) => false; + + /// + /// Armor-shaped pre-body hook: every module sees the incoming amount and may + /// scale it. Runs in module order before the OnDamage chain. + /// + public virtual long ModifyIncomingDamage(GameObject self, string damageType, long amount) => amount; + + /// + /// Death interception hook (SlowDeathBehavior-shaped). Returning true claims + /// the death: the object stays in the world and the module owns its eventual + /// removal. Returning false lets the next module try; if none claims it, the + /// object is removed at end of tick. + /// + public virtual bool OnDeath(SimWorld world, GameObject self) => false; + + public virtual void WriteState(CanonicalWriter writer) + { + } + + public virtual void ReadState(CanonicalReader reader) + { + } +} + +public sealed class ModuleRegistry +{ + private readonly SortedDictionary> _factories = new(StringComparer.Ordinal); + + public void Register(string typeName, Func factory) + { + _factories.Add(typeName, factory); + } + + public bool TryCreate(ModuleSpec spec, out ModuleBase? module) + { + if (_factories.TryGetValue(spec.TypeName, out var factory)) + { + module = factory(spec); + return true; + } + module = null; + return false; + } + + /// The P0/P1 vocabulary. Grows toward the measured ~350-type corpus tables. + public static ModuleRegistry CreateDefault() + { + var registry = new ModuleRegistry(); + registry.Register(ActiveBodyModule.TypeName, spec => new ActiveBodyModule(spec)); + registry.Register(ResourceGeneratorModule.TypeName, spec => new ResourceGeneratorModule(spec)); + registry.Register(LinearMoverModule.TypeName, spec => new LinearMoverModule(spec)); + registry.Register(SlowDeathModule.TypeName, spec => new SlowDeathModule(spec)); + registry.Register(ProductionModule.TypeName, spec => new ProductionModule(spec)); + registry.Register(GettingBuiltModule.TypeName, spec => new GettingBuiltModule(spec)); + registry.Register(StructureBodyModule.TypeName, spec => new StructureBodyModule(spec)); + registry.Register(ImmortalBodyModule.TypeName, spec => new ImmortalBodyModule(spec)); + registry.Register(LifetimeModule.TypeName, spec => new LifetimeModule(spec)); + registry.Register(DestroyDieModule.TypeName, spec => new DestroyDieModule(spec)); + registry.Register(KeepObjectDieModule.TypeName, spec => new KeepObjectDieModule(spec)); + registry.Register(StructureCollapseModule.TypeName, spec => new StructureCollapseModule(spec)); + registry.Register(ArmorModule.TypeName, spec => new ArmorModule(spec)); + registry.Register(SquishCollideModule.TypeName, spec => new SquishCollideModule(spec)); + registry.Register(WeaponModule.TypeName, spec => new WeaponModule(spec)); + registry.Register(AiCombatModule.TypeName, spec => new AiCombatModule(spec)); + registry.Register(HordeContainModule.TypeName, spec => new HordeContainModule(spec)); + registry.Register(BezierProjectileModule.TypeName, spec => new BezierProjectileModule(spec)); + registry.Register(AttributeModifierAuraModule.TypeName, spec => new AttributeModifierAuraModule(spec)); + return registry; + } +} + +/// +/// Health container in the shape of SAGE's ActiveBody (1,771 objects in the +/// measured corpus). P0 scope: max health from design data, damage intake, +/// death flagging for the world's removal sweep. +/// +public sealed class ActiveBodyModule : ModuleBase +{ + public const string TypeName = "ActiveBody"; + + public long Health { get; private set; } + public long MaxHealth { get; } + + public ActiveBodyModule(ModuleSpec spec) : base(spec) + { + MaxHealth = spec.GetLong("MaxHealth", 100); + Health = MaxHealth; + } + + public override bool OnDamage(SimWorld world, GameObject self, long amount) + { + if (amount < 0) + { + throw new ArgumentOutOfRangeException(nameof(amount), "Damage must be non-negative"); + } + if (Health == 0) + { + return true; + } + Health = Math.Max(0, Health - amount); + if (Health == 0) + { + world.HandleDeath(self); + } + return true; + } + + public override void WriteState(CanonicalWriter writer) => writer.WriteLong(Health); + public override void ReadState(CanonicalReader reader) => Health = reader.ReadLong(); +} + +/// +/// Periodic team income in the shape of TerrainResourceBehavior (152 objects in +/// the corpus): every IntervalTicks, credit Amount to the owner's resources. +/// +public sealed class ResourceGeneratorModule : ModuleBase +{ + public const string TypeName = "ResourceGenerator"; + + private readonly int _intervalTicks; + private readonly long _amount; + private int _ticksUntilPayout; + + public ResourceGeneratorModule(ModuleSpec spec) : base(spec) + { + _intervalTicks = (int)Math.Max(1, spec.GetLong("IntervalTicks", 30)); + _amount = spec.GetLong("Amount", 10); + _ticksUntilPayout = _intervalTicks; + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (self.IsUnderConstruction || self.IsDying) + { + return; + } + _ticksUntilPayout--; + if (_ticksUntilPayout > 0) + { + return; + } + _ticksUntilPayout = _intervalTicks; + world.AddTeamResources(self.Team, _amount); + } + + public override void WriteState(CanonicalWriter writer) => writer.WriteInt(_ticksUntilPayout); + public override void ReadState(CanonicalReader reader) => _ticksUntilPayout = reader.ReadInt(); +} + +/// +/// Minimal straight-line locomotion: moves toward Destination at Speed units per +/// tick. Stands in for the locomotor lane until P2 brings pathing; exists in P0 +/// so the determinism gates exercise continuous fixed-point state every tick. +/// +public sealed class LinearMoverModule : ModuleBase +{ + public const string TypeName = "LinearMover"; + + private readonly Fixed64 _speedPerTick; + private FixedVector2 _destination; + private bool _moving; + + public LinearMoverModule(ModuleSpec spec) : base(spec) + { + _speedPerTick = spec.GetFixed("SpeedPerTickRaw", Fixed64.FromFraction(1, 10)); + } + + public void SetDestination(FixedVector2 destination) + { + _destination = destination; + _moving = true; + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (!_moving) + { + return; + } + var offset = _destination - self.Position; + var distanceSquared = offset.LengthSquared(); + if (distanceSquared == Fixed64.Zero) + { + // Below Q32.32 squared-length precision the residual offset can be + // nonzero yet unrepresentable as progress — snap instead of stalling + // a hair short of the destination. + self.SetPosition(_destination); + _moving = false; + return; + } + var distance = Fixed64.Sqrt(distanceSquared); + if (distance <= _speedPerTick) + { + self.SetPosition(_destination); + _moving = false; + return; + } + var step = offset * (_speedPerTick / distance); + self.SetPosition(self.Position + step); + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteVector(_destination); + writer.WriteBool(_moving); + } + + public override void ReadState(CanonicalReader reader) + { + _destination = reader.ReadVector(); + _moving = reader.ReadBool(); + } +} + +/// +/// SlowDeathBehavior-shaped death interception (1,194 objects in the corpus): +/// claims the death, keeps the object in the world for DeathTicks (presentation +/// plays the death sequence there), then releases it for removal. +/// +public sealed class SlowDeathModule : ModuleBase +{ + public const string TypeName = "SlowDeath"; + + private readonly int _deathTicks; + private int _ticksRemaining; + private bool _dying; + + public SlowDeathModule(ModuleSpec spec) : base(spec) + { + _deathTicks = (int)Math.Max(1, spec.GetLong("DeathTicks", 30)); + } + + public bool IsDying => _dying; + + public override bool OnDeath(SimWorld world, GameObject self) + { + if (_dying) + { + return true; + } + _dying = true; + _ticksRemaining = _deathTicks; + self.MarkDying(); + return true; + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (!_dying) + { + return; + } + _ticksRemaining--; + if (_ticksRemaining <= 0) + { + self.MarkDead(); + } + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteBool(_dying); + writer.WriteInt(_ticksRemaining); + } + + public override void ReadState(CanonicalReader reader) + { + _dying = reader.ReadBool(); + _ticksRemaining = reader.ReadInt(); + } +} + +/// +/// ProductionUpdate-shaped queue (385 objects in the corpus). Design data encodes +/// buildable entries as "Build:template-name" = build ticks and optional +/// "Cost:template-name" = resource cost (default 0). TryQueue debits the owning +/// team's resources and rejects unaffordable requests; TryCancel refunds. +/// +/// Spawn phase matches the retail sim (dual-run oracle finding): a unit queued +/// by a command applied at tick T spawns DURING tick T + build_ticks — build +/// ticks T .. T+build_ticks-1 count down, and the completed head spawns on the +/// following update. Optional "RallyXRaw"/"RallyYRaw" design data (producer- +/// relative, Fixed64 raw) sends a spawned unit with a LinearMover walking from +/// the exit point to the rally point — the retail door-walk shape. +/// +public sealed class ProductionModule : ModuleBase +{ + public const string TypeName = "Production"; + + public const int MaxQueueLength = 9; + + private readonly FixedVector2 _exitOffset; + private readonly List<(string Template, int TicksRemaining)> _queue = new(); + + public ProductionModule(ModuleSpec spec) : base(spec) + { + _exitOffset = new FixedVector2( + spec.GetFixed("ExitOffsetXRaw", Fixed64.FromInt(2)), + spec.GetFixed("ExitOffsetYRaw", Fixed64.Zero)); + } + + public int QueueLength => _queue.Count; + + public long CostOf(string templateName) => Math.Max(0, Spec.GetLong("Cost:" + templateName, 0)); + + public bool TryQueue(SimWorld world, GameObject self, string templateName) + { + var buildTicks = Spec.GetLong("Build:" + templateName, -1); + if (buildTicks <= 0 || _queue.Count >= MaxQueueLength) + { + return false; + } + var cost = CostOf(templateName); + if (cost > 0) + { + if (world.TeamResources(self.Team) < cost) + { + return false; + } + world.AddTeamResources(self.Team, -cost); + } + _queue.Add((templateName, (int)buildTicks)); + return true; + } + + /// Cancels the queue entry at and refunds its cost. + public bool TryCancel(SimWorld world, GameObject self, int index) + { + if (index < 0 || index >= _queue.Count) + { + return false; + } + var (template, _) = _queue[index]; + _queue.RemoveAt(index); + world.AddTeamResources(self.Team, CostOf(template)); + return true; + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (self.IsUnderConstruction || self.IsDying || _queue.Count == 0) + { + return; + } + var (template, ticksRemaining) = _queue[0]; + if (ticksRemaining > 0) + { + _queue[0] = (template, ticksRemaining - 1); + return; + } + // ticksRemaining == 0: the head completed last tick; spawn this tick + // (command_tick + build_ticks). The next entry starts counting next tick. + _queue.RemoveAt(0); + var spawned = world.SpawnObject(template, self.Team, self.Position + _exitOffset); + if (Spec.Data.ContainsKey("RallyXRaw") || Spec.Data.ContainsKey("RallyYRaw")) + { + var rally = self.Position + new FixedVector2( + Spec.GetFixed("RallyXRaw", Fixed64.Zero), + Spec.GetFixed("RallyYRaw", Fixed64.Zero)); + spawned.FindModule()?.SetDestination(rally); + } + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteInt(_queue.Count); + foreach (var (template, ticksRemaining) in _queue) + { + writer.WriteString(template); + writer.WriteInt(ticksRemaining); + } + } + + public override void ReadState(CanonicalReader reader) + { + _queue.Clear(); + var count = reader.ReadInt(); + for (var i = 0; i < count; i++) + { + var template = reader.ReadString(); + var ticksRemaining = reader.ReadInt(); + _queue.Add((template, ticksRemaining)); + } + } +} + +/// +/// StructureBody variant of ActiveBody (373 objects): identical health +/// semantics today; exists as its own type so templates map 1:1 to the SAGE +/// vocabulary and structure-specific armor rules have a home in P2. +/// +public sealed class StructureBodyModule : ModuleBase +{ + public const string TypeName = "StructureBody"; + + public long Health { get; private set; } + public long MaxHealth { get; } + + public StructureBodyModule(ModuleSpec spec) : base(spec) + { + MaxHealth = spec.GetLong("MaxHealth", 500); + Health = MaxHealth; + } + + public override bool OnDamage(SimWorld world, GameObject self, long amount) + { + if (amount < 0) + { + throw new ArgumentOutOfRangeException(nameof(amount), "Damage must be non-negative"); + } + if (Health == 0) + { + return true; + } + Health = Math.Max(0, Health - amount); + if (Health == 0) + { + world.HandleDeath(self); + } + return true; + } + + public override void WriteState(CanonicalWriter writer) => writer.WriteLong(Health); + public override void ReadState(CanonicalReader reader) => Health = reader.ReadLong(); +} + +/// ImmortalBody (229 objects): takes damage down to 1 health, never dies. +public sealed class ImmortalBodyModule : ModuleBase +{ + public const string TypeName = "ImmortalBody"; + + public long Health { get; private set; } + public long MaxHealth { get; } + + public ImmortalBodyModule(ModuleSpec spec) : base(spec) + { + MaxHealth = spec.GetLong("MaxHealth", 100); + Health = MaxHealth; + } + + public override bool OnDamage(SimWorld world, GameObject self, long amount) + { + if (amount < 0) + { + throw new ArgumentOutOfRangeException(nameof(amount), "Damage must be non-negative"); + } + Health = Math.Max(1, Health - amount); + return true; + } + + public override void WriteState(CanonicalWriter writer) => writer.WriteLong(Health); + public override void ReadState(CanonicalReader reader) => Health = reader.ReadLong(); +} + +/// +/// LifetimeUpdate (198 objects): the object expires after LifetimeTicks, +/// routed through the normal death pipeline so SlowDeath-class modules can +/// still claim the removal. +/// +public sealed class LifetimeModule : ModuleBase +{ + public const string TypeName = "Lifetime"; + + private int _ticksRemaining; + private bool _expired; + + public LifetimeModule(ModuleSpec spec) : base(spec) + { + _ticksRemaining = (int)Math.Max(1, spec.GetLong("LifetimeTicks", 300)); + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (_expired || self.IsDying) + { + return; + } + _ticksRemaining--; + if (_ticksRemaining <= 0) + { + _expired = true; + world.HandleDeath(self); + } + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteInt(_ticksRemaining); + writer.WriteBool(_expired); + } + + public override void ReadState(CanonicalReader reader) + { + _ticksRemaining = reader.ReadInt(); + _expired = reader.ReadBool(); + } +} + +/// +/// Shared shape for death-claiming modules that hold the object for a timer +/// (SlowDeath, StructureCollapse, KeepObjectDie). Subclasses differ only in +/// type name, default duration, and whether zero duration means forever. +/// +public abstract class TimedDeathModuleBase : ModuleBase +{ + private readonly int _holdTicks; + private readonly bool _zeroMeansForever; + private int _ticksRemaining; + private bool _dying; + + protected TimedDeathModuleBase(ModuleSpec spec, string dataKey, long defaultTicks, bool zeroMeansForever) + : base(spec) + { + var configured = spec.GetLong(dataKey, defaultTicks); + _zeroMeansForever = zeroMeansForever && configured == 0; + _holdTicks = (int)Math.Max(_zeroMeansForever ? 0 : 1, configured); + } + + public bool IsDying => _dying; + + public override bool OnDeath(SimWorld world, GameObject self) + { + if (_dying) + { + return true; + } + _dying = true; + _ticksRemaining = _holdTicks; + self.MarkDying(); + return true; + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (!_dying || _zeroMeansForever) + { + return; + } + _ticksRemaining--; + if (_ticksRemaining <= 0) + { + self.MarkDead(); + } + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteBool(_dying); + writer.WriteInt(_ticksRemaining); + } + + public override void ReadState(CanonicalReader reader) + { + _dying = reader.ReadBool(); + _ticksRemaining = reader.ReadInt(); + } +} + +/// DestroyDie (385 objects): explicit immediate removal on death. +public sealed class DestroyDieModule : ModuleBase +{ + public const string TypeName = "DestroyDie"; + + public DestroyDieModule(ModuleSpec spec) : base(spec) + { + } + + public override bool OnDeath(SimWorld world, GameObject self) + { + self.MarkDead(); + return true; + } +} + +/// +/// KeepObjectDie (268 objects): the corpse stays in the world — forever when +/// KeepTicks is 0 (cleanup lanes come later), else for KeepTicks. +/// +public sealed class KeepObjectDieModule : TimedDeathModuleBase +{ + public const string TypeName = "KeepObjectDie"; + + public KeepObjectDieModule(ModuleSpec spec) + : base(spec, "KeepTicks", 0, zeroMeansForever: true) + { + } +} + +/// +/// StructureCollapseUpdate (656 objects): the collapsing building holds for +/// its rubble sequence, then releases for removal. +/// +public sealed class StructureCollapseModule : TimedDeathModuleBase +{ + public const string TypeName = "StructureCollapse"; + + public StructureCollapseModule(ModuleSpec spec) + : base(spec, "CollapseTicks", 45, zeroMeansForever: false) + { + } +} + +/// +/// GettingBuiltBehavior-shaped construction gate (350 objects in the corpus): +/// the object spawns under construction and other economic modules idle until +/// the build completes. P1 scope: time-driven completion; builder escorting +/// arrives with the worker lane. +/// +public sealed class GettingBuiltModule : ModuleBase +{ + public const string TypeName = "GettingBuilt"; + + private int _ticksRemaining; + private bool _started; + + public GettingBuiltModule(ModuleSpec spec) : base(spec) + { + _ticksRemaining = (int)Math.Max(1, spec.GetLong("ConstructionTicks", 60)); + } + + public override void OnUpdate(SimWorld world, GameObject self) + { + if (!_started) + { + _started = true; + self.SetUnderConstruction(true); + } + if (!self.IsUnderConstruction) + { + return; + } + _ticksRemaining--; + if (_ticksRemaining <= 0) + { + self.SetUnderConstruction(false); + } + } + + public override void WriteState(CanonicalWriter writer) + { + writer.WriteBool(_started); + writer.WriteInt(_ticksRemaining); + } + + public override void ReadState(CanonicalReader reader) + { + _started = reader.ReadBool(); + _ticksRemaining = reader.ReadInt(); + } +} diff --git a/engine/OpenBfme.Sim/OpenBfme.Sim.csproj b/engine/OpenBfme.Sim/OpenBfme.Sim.csproj new file mode 100644 index 0000000..7b1b112 --- /dev/null +++ b/engine/OpenBfme.Sim/OpenBfme.Sim.csproj @@ -0,0 +1,12 @@ + + + + net8.0 + enable + enable + latest + true + OpenBfme.Sim + + + diff --git a/engine/OpenBfme.Sim/PackTemplateLoader.cs b/engine/OpenBfme.Sim/PackTemplateLoader.cs new file mode 100644 index 0000000..2fedb12 --- /dev/null +++ b/engine/OpenBfme.Sim/PackTemplateLoader.cs @@ -0,0 +1,371 @@ +using System.Globalization; +using System.Numerics; +using System.Text.Json; + +namespace OpenBfme.Sim; + +/// +/// Loads ObjectTemplates from a content pack's data/objects.json document +/// (schema "openbfme.objects", rows like bfme2.object.gondor-fighter). +/// +/// FLOAT-AVOIDANCE STRATEGY (the hard determinism requirement): +/// numeric JSON is never read through GetDouble/GetSingle. Every number is +/// taken as its raw JSON text (JsonElement.GetRawText()), parsed with +/// decimal.Parse(InvariantCulture) — which is exact base-10, no binary +/// float round-trip — then decomposed via decimal.GetBits into an exact +/// integer numerator and power-of-ten denominator. That rational feeds +/// Fixed64.FromFraction (itself BigInteger-based). The same JSON text +/// therefore yields bit-identical Fixed64 raw values on every platform. +/// No float/double arithmetic exists anywhere on the loader path. +/// +/// STRUCTURE-VS-UNIT PREDICATE: the pack row's "kind" field is authoritative. +/// kind == "structure" gets a StructureBody module; the mobile kinds +/// ("member", "builder", "battalion") get an ActiveBody module. Any other +/// kind is skipped fail-closed with a typed reason — never guessed. +/// +public static class PackTemplateLoader +{ + public const string ExpectedSchema = "openbfme.objects"; + + /// Ticks used to convert pack speed (units/second) to per-tick movement. + public const int TicksPerSecond = SimWorld.TicksPerSecond; + + private static readonly string[] MobileKinds = { "member", "builder", "battalion" }; + + public static PackTemplateLoadResult LoadFromObjectsDocument(string json) + { + JsonDocument document; + try + { + document = JsonDocument.Parse(json); + } + catch (JsonException exception) + { + throw new PackObjectsDocumentException("objects document is not valid JSON", exception); + } + + using (document) + { + var root = document.RootElement; + if (root.ValueKind != JsonValueKind.Object) + { + throw new PackObjectsDocumentException("objects document root is not a JSON object"); + } + if (root.TryGetProperty("schema", out var schema) + && (schema.ValueKind != JsonValueKind.String || schema.GetString() != ExpectedSchema)) + { + throw new PackObjectsDocumentException( + $"objects document schema is '{schema}' (expected '{ExpectedSchema}')"); + } + if (!root.TryGetProperty("objects", out var rows) || rows.ValueKind != JsonValueKind.Array) + { + throw new PackObjectsDocumentException("objects document has no 'objects' array"); + } + + var templates = new List(); + var seenIds = new HashSet(StringComparer.Ordinal); + var skipped = new List(); + var unmapped = new SortedDictionary(StringComparer.Ordinal); + var notes = new List(); + + var index = -1; + foreach (var row in rows.EnumerateArray()) + { + index++; + LoadRow(row, index, templates, seenIds, skipped, unmapped, notes); + } + + var report = new LoadReport(templates.Count, skipped, unmapped, notes); + return new PackTemplateLoadResult(templates, report); + } + } + + private static void LoadRow( + JsonElement row, + int index, + List templates, + HashSet seenIds, + List skipped, + SortedDictionary unmapped, + List notes) + { + if (row.ValueKind != JsonValueKind.Object) + { + skipped.Add(new SkippedRow(index, "", RowSkipReason.NotAnObject, $"row kind is {row.ValueKind}")); + return; + } + if (!row.TryGetProperty("id", out var idElement) || idElement.ValueKind != JsonValueKind.String + || string.IsNullOrEmpty(idElement.GetString())) + { + skipped.Add(new SkippedRow(index, "", RowSkipReason.MissingId, "row has no non-empty string 'id'")); + return; + } + var id = idElement.GetString()!; + if (!row.TryGetProperty("kind", out var kindElement) || kindElement.ValueKind != JsonValueKind.String) + { + skipped.Add(new SkippedRow(index, id, RowSkipReason.MissingKind, "row has no string 'kind'")); + return; + } + var kind = kindElement.GetString()!; + var isStructure = kind == "structure"; + if (!isStructure && Array.IndexOf(MobileKinds, kind) < 0) + { + skipped.Add(new SkippedRow(index, id, RowSkipReason.UnknownKind, $"kind '{kind}' is not a known object kind")); + return; + } + if (!seenIds.Add(id)) + { + skipped.Add(new SkippedRow(index, id, RowSkipReason.DuplicateId, "a previous row already used this id")); + return; + } + + var bodyData = new SortedDictionary(StringComparer.Ordinal); + var bodyStrings = new SortedDictionary(StringComparer.Ordinal); + Fixed64? speedPerTick = null; + var invalid = false; + + foreach (var property in row.EnumerateObject()) + { + switch (property.Name) + { + case "id": + case "kind": + break; // consumed above + case "displayName": + if (property.Value.ValueKind == JsonValueKind.String) + { + bodyStrings["DisplayName"] = property.Value.GetString()!; + } + else + { + CountUnmapped(unmapped, "displayName"); + } + break; + case "memberObjectId": + if (property.Value.ValueKind == JsonValueKind.String) + { + bodyStrings["MemberObjectId"] = property.Value.GetString()!; + } + else + { + CountUnmapped(unmapped, "memberObjectId"); + } + break; + case "memberCount": + invalid |= !TryMapPositiveLong(property.Value, "memberCount", "MemberCount", bodyData, id, index, skipped); + break; + case "commandPoints": + invalid |= !TryMapPositiveLong(property.Value, "commandPoints", "CommandPoints", bodyData, id, index, skipped); + break; + case "simulation": + if (property.Value.ValueKind != JsonValueKind.Object) + { + skipped.Add(new SkippedRow(index, id, RowSkipReason.InvalidNumericField, "'simulation' is not an object")); + invalid = true; + break; + } + foreach (var simProperty in property.Value.EnumerateObject()) + { + switch (simProperty.Name) + { + case "health": + invalid |= !TryMapPositiveLong(simProperty.Value, "simulation.health", "MaxHealth", bodyData, id, index, skipped); + break; + case "speed": + if (TryReadFraction(simProperty.Value, out var numerator, out var denominator) + && numerator > 0 && denominator <= long.MaxValue / TicksPerSecond) + { + // Pack speed is units/second; the sim moves per tick. + speedPerTick = Fixed64.FromFraction(numerator, denominator * TicksPerSecond); + } + else + { + skipped.Add(new SkippedRow(index, id, RowSkipReason.InvalidNumericField, + $"'simulation.speed' is not a positive number: {simProperty.Value.GetRawText()}")); + invalid = true; + } + break; + default: + CountUnmapped(unmapped, "simulation." + simProperty.Name); + break; + } + if (invalid) + { + break; + } + } + break; + default: + // presentation, animationCapabilityId, sourceTypeName, formations, ... + CountUnmapped(unmapped, property.Name); + break; + } + if (invalid) + { + break; + } + } + if (invalid) + { + return; + } + + if (!bodyData.ContainsKey("MaxHealth")) + { + notes.Add($"{id}: no simulation.health field; {(isStructure ? StructureBodyModule.TypeName : ActiveBodyModule.TypeName)} uses its module default"); + } + + var modules = new List + { + new(isStructure ? StructureBodyModule.TypeName : ActiveBodyModule.TypeName, bodyData, bodyStrings), + }; + if (speedPerTick is { } speed) + { + modules.Add(new ModuleSpec(LinearMoverModule.TypeName, new Dictionary + { + ["SpeedPerTickRaw"] = speed.Raw, + })); + } + templates.Add(new ObjectTemplate(id, modules)); + } + + private static bool TryMapPositiveLong( + JsonElement element, + string fieldName, + string dataKey, + SortedDictionary data, + string id, + int index, + List skipped) + { + if (TryReadFraction(element, out var numerator, out var denominator) + && denominator == 1 && numerator > 0) + { + data[dataKey] = numerator; + return true; + } + skipped.Add(new SkippedRow(index, id, RowSkipReason.InvalidNumericField, + $"'{fieldName}' is not a positive integer: {(element.ValueKind == JsonValueKind.Number ? element.GetRawText() : element.ValueKind.ToString())}")); + return false; + } + + /// + /// Exact rational read of a JSON number: raw text -> decimal (exact base-10) + /// -> integer numerator over power-of-ten denominator, reduced. Returns false + /// for non-numbers and values that cannot fit the long-based rational. + /// Never touches float/double. + /// + private static bool TryReadFraction(JsonElement element, out long numerator, out long denominator) + { + numerator = 0; + denominator = 1; + if (element.ValueKind != JsonValueKind.Number) + { + return false; + } + decimal value; + try + { + value = decimal.Parse(element.GetRawText(), NumberStyles.Float, CultureInfo.InvariantCulture); + } + catch (OverflowException) + { + return false; + } + catch (FormatException) + { + return false; + } + var bits = decimal.GetBits(value); + var scale = (bits[3] >> 16) & 0xFF; + var negative = (bits[3] & unchecked((int)0x80000000)) != 0; + var magnitude = ((BigInteger)(uint)bits[2] << 64) + | ((BigInteger)(uint)bits[1] << 32) + | (uint)bits[0]; + var num = negative ? -magnitude : magnitude; + var den = BigInteger.Pow(10, scale); + if (!num.IsZero) + { + var gcd = BigInteger.GreatestCommonDivisor(BigInteger.Abs(num), den); + num /= gcd; + den /= gcd; + } + else + { + den = BigInteger.One; + } + if (num > long.MaxValue || num < long.MinValue || den > long.MaxValue) + { + return false; + } + numerator = (long)num; + denominator = (long)den; + return true; + } + + private static void CountUnmapped(SortedDictionary unmapped, string field) => + unmapped[field] = unmapped.TryGetValue(field, out var count) ? count + 1 : 1; +} + +/// Templates plus the fail-closed accounting for one load pass. +public sealed class PackTemplateLoadResult +{ + public IReadOnlyList Templates { get; } + public LoadReport Report { get; } + + public PackTemplateLoadResult(IReadOnlyList templates, LoadReport report) + { + Templates = templates; + Report = report; + } +} + +/// +/// Fail-closed load accounting: every row that did not become a template is in +/// SkippedRows with a typed reason; every source field the loader saw but does +/// not map is enumerated (with occurrence counts) in UnmappedFields; defaults +/// applied in place of absent data are spelled out in Notes. Nothing is silent. +/// +public sealed class LoadReport +{ + public int LoadedCount { get; } + public IReadOnlyList SkippedRows { get; } + public IReadOnlyDictionary UnmappedFields { get; } + public IReadOnlyList Notes { get; } + + public LoadReport( + int loadedCount, + IReadOnlyList skippedRows, + IReadOnlyDictionary unmappedFields, + IReadOnlyList notes) + { + LoadedCount = loadedCount; + SkippedRows = skippedRows; + UnmappedFields = unmappedFields; + Notes = notes; + } +} + +public enum RowSkipReason +{ + NotAnObject, + MissingId, + DuplicateId, + MissingKind, + UnknownKind, + InvalidNumericField, +} + +public sealed record SkippedRow(int Index, string Id, RowSkipReason Reason, string Detail); + +/// Typed error for a structurally unusable objects document. +public sealed class PackObjectsDocumentException : Exception +{ + public PackObjectsDocumentException(string message) : base(message) + { + } + + public PackObjectsDocumentException(string message, Exception inner) : base(message, inner) + { + } +} diff --git a/engine/OpenBfme.Sim/SimCommand.cs b/engine/OpenBfme.Sim/SimCommand.cs new file mode 100644 index 0000000..a764f36 --- /dev/null +++ b/engine/OpenBfme.Sim/SimCommand.cs @@ -0,0 +1,142 @@ +namespace OpenBfme.Sim; + +/// +/// The single mutation entry point for the simulation, mirroring the GDScript +/// lockstep contract: commands are scheduled for a future tick and applied at +/// tick start ordered by (team, seq). Args hold only canonical value types. +/// +public sealed class SimCommand +{ + public int Tick { get; } + public int Team { get; } + public int Seq { get; } + public string Type { get; } + public IReadOnlyDictionary Args => _args; + + private readonly SortedDictionary _args; + + public SimCommand(int tick, int team, int seq, string type, IEnumerable>? args = null) + { + if (tick < 0) + { + throw new ArgumentOutOfRangeException(nameof(tick)); + } + if (team < 0) + { + throw new ArgumentOutOfRangeException(nameof(team)); + } + if (seq < 0) + { + throw new ArgumentOutOfRangeException(nameof(seq)); + } + Tick = tick; + Team = team; + Seq = seq; + Type = type ?? throw new ArgumentNullException(nameof(type)); + _args = new SortedDictionary(StringComparer.Ordinal); + if (args != null) + { + foreach (var pair in args) + { + _args.Add(pair.Key, pair.Value); + } + } + } + + public long GetLong(string key) => Args.TryGetValue(key, out var value) && value.Kind == CommandValueKind.Long + ? value.LongValue + : throw new KeyNotFoundException($"Command '{Type}' missing long arg '{key}'"); + + public Fixed64 GetFixed(string key) => Args.TryGetValue(key, out var value) && value.Kind == CommandValueKind.Fixed + ? Fixed64.FromRaw(value.LongValue) + : throw new KeyNotFoundException($"Command '{Type}' missing fixed arg '{key}'"); + + public string GetString(string key) => Args.TryGetValue(key, out var value) && value.Kind == CommandValueKind.String + ? value.StringValue! + : throw new KeyNotFoundException($"Command '{Type}' missing string arg '{key}'"); + + internal void WriteTo(CanonicalWriter writer) + { + writer.WriteInt(Tick); + writer.WriteInt(Team); + writer.WriteInt(Seq); + writer.WriteString(Type); + writer.WriteInt(_args.Count); + foreach (var (key, value) in _args) + { + writer.WriteString(key); + value.WriteTo(writer); + } + } + + internal static SimCommand ReadFrom(CanonicalReader reader) + { + var tick = reader.ReadInt(); + var team = reader.ReadInt(); + var seq = reader.ReadInt(); + var type = reader.ReadString(); + var count = reader.ReadInt(); + var args = new List>(count); + for (var i = 0; i < count; i++) + { + var key = reader.ReadString(); + args.Add(new KeyValuePair(key, CommandValue.ReadFrom(reader))); + } + return new SimCommand(tick, team, seq, type, args); + } +} + +public enum CommandValueKind : byte +{ + Long = 1, + Fixed = 2, + String = 3, +} + +/// Canonical command argument: integer, fixed-point scalar, or string. Nothing else. +public readonly struct CommandValue +{ + public CommandValueKind Kind { get; } + public long LongValue { get; } + public string? StringValue { get; } + + private CommandValue(CommandValueKind kind, long longValue, string? stringValue) + { + Kind = kind; + LongValue = longValue; + StringValue = stringValue; + } + + public static CommandValue OfLong(long value) => new(CommandValueKind.Long, value, null); + public static CommandValue OfFixed(Fixed64 value) => new(CommandValueKind.Fixed, value.Raw, null); + public static CommandValue OfString(string value) => new(CommandValueKind.String, 0, value ?? throw new ArgumentNullException(nameof(value))); + + internal void WriteTo(CanonicalWriter writer) + { + writer.WriteByte((byte)Kind); + switch (Kind) + { + case CommandValueKind.Long: + case CommandValueKind.Fixed: + writer.WriteLong(LongValue); + break; + case CommandValueKind.String: + writer.WriteString(StringValue!); + break; + default: + throw new InvalidOperationException($"Unserializable command value kind {Kind}"); + } + } + + internal static CommandValue ReadFrom(CanonicalReader reader) + { + var kind = (CommandValueKind)reader.ReadByte(); + return kind switch + { + CommandValueKind.Long => OfLong(reader.ReadLong()), + CommandValueKind.Fixed => OfFixed(Fixed64.FromRaw(reader.ReadLong())), + CommandValueKind.String => OfString(reader.ReadString()), + _ => throw new InvalidDataException($"Unknown command value kind {kind}"), + }; + } +} diff --git a/engine/OpenBfme.Sim/SimWorld.cs b/engine/OpenBfme.Sim/SimWorld.cs new file mode 100644 index 0000000..0ce2642 --- /dev/null +++ b/engine/OpenBfme.Sim/SimWorld.cs @@ -0,0 +1,471 @@ +namespace OpenBfme.Sim; + +/// Immutable per-match configuration: templates, seed, teams. Not part of the state hash. +public sealed class SimConfig +{ + public IReadOnlyDictionary Templates { get; } + public ulong RandomSeed { get; } + public int TeamCount { get; } + + public SimConfig(IEnumerable templates, ulong randomSeed, int teamCount) + { + if (teamCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(teamCount)); + } + var map = new SortedDictionary(StringComparer.Ordinal); + foreach (var template in templates) + { + map.Add(template.Name, template); + } + Templates = map; + RandomSeed = randomSeed; + TeamCount = teamCount; + } +} + +/// +/// The deterministic simulation world: fixed integer ticks, command-queue +/// mutation only, canonical hash + snapshot. This is the P0 kernel the module +/// vocabulary grows into. +/// +public sealed class SimWorld +{ + public const int TicksPerSecond = 30; + + private readonly SimConfig _config; + private readonly ModuleRegistry _registry; + private readonly SortedDictionary _objects = new(); + private readonly SortedDictionary> _pendingCommands = new(); + private readonly long[] _teamResources; + private readonly SortedDictionary _moduleGaps = new(StringComparer.Ordinal); + private DeterministicRandom _random; + private int _nextObjectId = 1; + private bool _inUpdateSweep; + private readonly List _pendingSpawns = new(); + // Aura armor table: summed basis points of incoming-damage reduction per + // object id. DERIVED state — rebuilt at end of every tick (and after + // Restore) from AttributeModifierAuraModule caches in ascending carrier id + // order, so it is deliberately NOT serialized or hashed. + private readonly SortedDictionary _auraArmorBonusBp = new(); + + public int TickIndex { get; private set; } + public IReadOnlyDictionary Objects => _objects; + /// Module type names that had no registered implementation, with occurrence counts. Fail-closed accounting. + public IReadOnlyDictionary ModuleGaps => _moduleGaps; + + public SimWorld(SimConfig config, ModuleRegistry registry) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + _registry = registry ?? throw new ArgumentNullException(nameof(registry)); + _teamResources = new long[config.TeamCount]; + _random = new DeterministicRandom(config.RandomSeed); + } + + public long TeamResources(int team) => _teamResources[ValidateTeam(team)]; + + public void AddTeamResources(int team, long amount) => _teamResources[ValidateTeam(team)] += amount; + + public GameObject SpawnObject(string templateName, int team, FixedVector2 position) + { + ValidateTeam(team); + if (!_config.Templates.TryGetValue(templateName, out var template)) + { + throw new KeyNotFoundException($"Unknown object template '{templateName}'"); + } + var modules = new List(template.Modules.Count); + foreach (var spec in template.Modules) + { + if (_registry.TryCreate(spec, out var module)) + { + modules.Add(module!); + } + else + { + _moduleGaps[spec.TypeName] = _moduleGaps.TryGetValue(spec.TypeName, out var count) ? count + 1 : 1; + } + } + var gameObject = new GameObject(_nextObjectId++, templateName, team, position, modules); + if (_inUpdateSweep) + { + // Spawns requested by modules mid-sweep (production, death rubble) + // are deferred so scanning modules never see the object dictionary + // mutate under them; newcomers join at end of sweep, first update + // next tick — deterministically. + _pendingSpawns.Add(gameObject); + } + else + { + _objects.Add(gameObject.Id, gameObject); + } + return gameObject; + } + + public bool SubmitCommand(SimCommand command) + { + if (command.Tick <= TickIndex) + { + return false; + } + if (command.Team < 0 || command.Team >= _config.TeamCount) + { + return false; + } + if (!_pendingCommands.TryGetValue(command.Tick, out var list)) + { + list = new List(); + _pendingCommands.Add(command.Tick, list); + } + list.Add(command); + return true; + } + + public void Tick() + { + TickIndex++; + ApplyPendingCommands(); + // The object dictionary is frozen for the whole sweep: mid-sweep spawns + // divert to _pendingSpawns (so modules scanning Objects never see it + // mutate) and join afterwards, first updating next tick. Dead objects + // never update. + var updateList = new List(_objects.Values); + _inUpdateSweep = true; + try + { + foreach (var gameObject in updateList) + { + if (gameObject.IsDead) + { + continue; + } + foreach (var module in gameObject.Modules) + { + module.OnUpdate(this, gameObject); + if (gameObject.IsDead) + { + break; + } + } + } + } + finally + { + _inUpdateSweep = false; + } + foreach (var spawned in _pendingSpawns) + { + _objects.Add(spawned.Id, spawned); + } + _pendingSpawns.Clear(); + RemoveDeadObjects(); + RebuildAuraTable(); + } + + /// + /// Rebuilds the aura armor table from every living, non-dying, constructed + /// carrier's cached member ids (ascending carrier id order — deterministic). + /// Runs at end of tick so the table is stable for the whole following tick. + /// + private void RebuildAuraTable() + { + _auraArmorBonusBp.Clear(); + foreach (var gameObject in _objects.Values) + { + if (gameObject.IsDying || gameObject.IsUnderConstruction) + { + continue; + } + foreach (var module in gameObject.Modules) + { + if (module is AttributeModifierAuraModule aura) + { + aura.ContributeTo(_auraArmorBonusBp, this); + } + } + } + } + + /// Summed aura armor basis points currently granted to an object (0 if none). + public long AuraArmorBonusBp(int objectId) => + _auraArmorBonusBp.TryGetValue(objectId, out var bp) ? bp : 0; + + public void Advance(int ticks) + { + for (var i = 0; i < ticks; i++) + { + Tick(); + } + } + + private void ApplyPendingCommands() + { + if (!_pendingCommands.TryGetValue(TickIndex, out var commands)) + { + return; + } + _pendingCommands.Remove(TickIndex); + commands.Sort(static (a, b) => + { + var byTeam = a.Team.CompareTo(b.Team); + return byTeam != 0 ? byTeam : a.Seq.CompareTo(b.Seq); + }); + foreach (var command in commands) + { + ApplyCommand(command); + } + } + + private void ApplyCommand(SimCommand command) + { + switch (command.Type) + { + case "spawn": + SpawnObject(command.GetString("template"), command.Team, + new FixedVector2(command.GetFixed("x"), command.GetFixed("y"))); + break; + case "move": + if (_objects.TryGetValue((int)command.GetLong("id"), out var mover) && mover.Team == command.Team) + { + mover.FindModule()?.SetDestination( + new FixedVector2(command.GetFixed("x"), command.GetFixed("y"))); + } + break; + case "damage": + if (_objects.TryGetValue((int)command.GetLong("id"), out var victim)) + { + DealDamage(victim, command.GetLong("amount")); + } + break; + case "set_resources": + _teamResources[command.Team] = command.GetLong("amount"); + break; + case "roll": + // Consumes randomness so tests prove the RNG stream is authoritative state. + _teamResources[command.Team] += _random.NextBelow(100); + break; + case "queue_production": + if (_objects.TryGetValue((int)command.GetLong("id"), out var producer) + && producer.Team == command.Team) + { + // Pass-through: cost debit/affordability live in TryQueue. + producer.FindModule()?.TryQueue(this, producer, command.GetString("template")); + } + break; + case "cancel_production": + if (_objects.TryGetValue((int)command.GetLong("id"), out var canceller) + && canceller.Team == command.Team) + { + canceller.FindModule()?.TryCancel(this, canceller, (int)command.GetLong("index")); + } + break; + default: + // Unknown command types are ignored deterministically (validated upstream + // by the lockstep layer); they still affected the hash while queued. + break; + } + } + + public void DealDamage(GameObject target, long amount) => DealDamage(target, amount, DamageTypes.Default); + + public void DealDamage(GameObject target, long amount, string damageType) + { + if (target.IsDead || target.IsDying) + { + return; + } + // Crush damage only lands on objects that declare themselves crushable. + if (damageType == DamageTypes.Crush && target.FindModule() == null) + { + return; + } + foreach (var module in target.Modules) + { + amount = module.ModifyIncomingDamage(target, damageType, amount); + } + // Aura armor applies after the per-module chain. Stacked contributions + // add; the sum is clamped to [0, 10000] bp (full immunity, never a heal). + var auraBp = Math.Clamp(AuraArmorBonusBp(target.Id), 0, 10_000); + if (auraBp > 0) + { + amount -= amount * auraBp / 10_000; + } + if (amount <= 0) + { + return; + } + foreach (var module in target.Modules) + { + if (module.OnDamage(this, target, amount)) + { + return; + } + } + } + + /// + /// Death pipeline: the first module claiming the death (SlowDeath-shaped) + /// owns removal timing; otherwise the object is removed at end of tick. + /// + public void HandleDeath(GameObject target) + { + foreach (var module in target.Modules) + { + if (module.OnDeath(this, target)) + { + return; + } + } + target.MarkDead(); + } + + private void RemoveDeadObjects() + { + List? deadIds = null; + foreach (var (id, gameObject) in _objects) + { + if (gameObject.IsDead) + { + (deadIds ??= new List()).Add(id); + } + } + if (deadIds == null) + { + return; + } + foreach (var id in deadIds) + { + _objects.Remove(id); + } + } + + private int ValidateTeam(int team) => + team >= 0 && team < _config.TeamCount + ? team + : throw new ArgumentOutOfRangeException(nameof(team), $"Team {team} outside 0..{_config.TeamCount - 1}"); + + private void WriteAuthoritativeState(CanonicalWriter writer) + { + writer.WriteInt(TickIndex); + writer.WriteInt(_nextObjectId); + writer.WriteInt(_teamResources.Length); + foreach (var resources in _teamResources) + { + writer.WriteLong(resources); + } + var (randomState, randomIncrement) = _random.Serialize(); + writer.WriteLong(unchecked((long)randomState)); + writer.WriteLong(unchecked((long)randomIncrement)); + writer.WriteInt(_objects.Count); + foreach (var gameObject in _objects.Values) + { + gameObject.WriteState(writer); + } + writer.WriteInt(_pendingCommands.Count); + foreach (var (tick, commands) in _pendingCommands) + { + writer.WriteInt(tick); + writer.WriteInt(commands.Count); + foreach (var command in commands) + { + command.WriteTo(writer); + } + } + } + + public string StateHash() + { + var writer = new CanonicalWriter(); + WriteAuthoritativeState(writer); + return writer.ToSha256Hex(); + } + + public byte[] Snapshot() + { + var writer = new CanonicalWriter(); + WriteAuthoritativeState(writer); + return writer.ToArray(); + } + + /// Reconstructs a world from a snapshot. Requires the same config and registry the snapshot was taken with. + public static SimWorld Restore(byte[] snapshot, SimConfig config, ModuleRegistry registry) + { + var world = new SimWorld(config, registry); + var reader = new CanonicalReader(snapshot); + world.TickIndex = reader.ReadInt(); + world._nextObjectId = reader.ReadInt(); + var teamCount = reader.ReadInt(); + if (teamCount != config.TeamCount) + { + throw new InvalidDataException($"Snapshot has {teamCount} teams, config has {config.TeamCount}"); + } + for (var i = 0; i < teamCount; i++) + { + world._teamResources[i] = reader.ReadLong(); + } + var randomState = unchecked((ulong)reader.ReadLong()); + var randomIncrement = unchecked((ulong)reader.ReadLong()); + world._random = DeterministicRandom.Deserialize(randomState, randomIncrement); + var objectCount = reader.ReadInt(); + for (var i = 0; i < objectCount; i++) + { + world.ReadObject(reader); + } + var pendingTickCount = reader.ReadInt(); + for (var i = 0; i < pendingTickCount; i++) + { + var tick = reader.ReadInt(); + var commandCount = reader.ReadInt(); + var commands = new List(commandCount); + for (var j = 0; j < commandCount; j++) + { + commands.Add(SimCommand.ReadFrom(reader)); + } + world._pendingCommands.Add(tick, commands); + } + reader.ExpectEnd(); + world.RebuildAuraTable(); // derived state: not in the snapshot, rebuilt from module caches + return world; + } + + private void ReadObject(CanonicalReader reader) + { + var id = reader.ReadInt(); + var templateName = reader.ReadString(); + var team = reader.ReadInt(); + var position = reader.ReadVector(); + var isDead = reader.ReadBool(); + var isDying = reader.ReadBool(); + var isUnderConstruction = reader.ReadBool(); + if (!_config.Templates.TryGetValue(templateName, out var template)) + { + throw new InvalidDataException($"Snapshot references unknown template '{templateName}'"); + } + var modules = new List(template.Modules.Count); + foreach (var spec in template.Modules) + { + if (_registry.TryCreate(spec, out var module)) + { + modules.Add(module!); + } + else + { + _moduleGaps[spec.TypeName] = _moduleGaps.TryGetValue(spec.TypeName, out var count) ? count + 1 : 1; + } + } + var gameObject = new GameObject(id, templateName, team, position, modules); + foreach (var module in modules) + { + module.ReadState(reader); + } + if (isDead) + { + gameObject.MarkDead(); + } + if (isDying) + { + gameObject.MarkDying(); + } + gameObject.SetUnderConstruction(isUnderConstruction); + _objects.Add(id, gameObject); + } +} diff --git a/game/export_presets.cfg b/game/export_presets.cfg new file mode 100644 index 0000000..9543c6e --- /dev/null +++ b/game/export_presets.cfg @@ -0,0 +1,74 @@ +[preset.0] + +name="windows" +platform="Windows Desktop" +runnable=true +advanced_options=false +dedicated_server=false +custom_features="" +export_filter="all_resources" +include_filter="" +exclude_filter="" +export_path="../dist/windows/OpenBFME.exe" +patches=PackedStringArray() +encryption_include_filters="" +encryption_exclude_filters="" +seed=0 +encrypt_pck=false +encrypt_directory=false +script_export_mode=2 + +[preset.0.options] + +custom_template/debug="" +custom_template/release="" +debug/export_console_wrapper=1 +binary_format/embed_pck=false +texture_format/s3tc_bptc=true +texture_format/etc2_astc=false +binary_format/architecture="x86_64" +codesign/enable=false +application/modify_resources=false +application/icon="" +application/console_wrapper_icon="" +application/icon_interpolation=4 +application/file_version="" +application/product_version="" +application/company_name="" +application/product_name="OpenBFME" +application/file_description="OpenBFME engine (no game assets included)" +application/copyright="" +application/trademarks="" +application/export_angle=0 +application/export_d3d12=0 +application/d3d12_agility_sdk_multiarch=true + +[preset.1] + +name="linux" +platform="Linux/X11" +runnable=true +advanced_options=false +dedicated_server=false +custom_features="" +export_filter="all_resources" +include_filter="" +exclude_filter="" +export_path="../dist/linux/OpenBFME.x86_64" +patches=PackedStringArray() +encryption_include_filters="" +encryption_exclude_filters="" +seed=0 +encrypt_pck=false +encrypt_directory=false +script_export_mode=2 + +[preset.1.options] + +custom_template/debug="" +custom_template/release="" +debug/export_console_wrapper=1 +binary_format/embed_pck=false +binary_format/architecture="x86_64" +texture_format/s3tc_bptc=true +texture_format/etc2_astc=false diff --git a/importer/blender/w3d_multi_to_glb.py b/importer/blender/w3d_multi_to_glb.py new file mode 100644 index 0000000..8aed387 --- /dev/null +++ b/importer/blender/w3d_multi_to_glb.py @@ -0,0 +1,220 @@ +"""Run multiple W3D→GLB jobs in one Blender process. + +Host stages inputs and writes a JSON job list. This driver initializes the +plugin once, converts each job via convert_w3d_job, and emits one marker per +job so the pipeline can keep its existing validation path. + +Each job runs with the process output descriptors redirected into per-job +files, and the captured text rides the success marker as ``output_log``. The +pipeline's warning-text guards (``not supported`` / ``texture not found``) +must evaluate the same real content the single-job process log carries — +a GLB with unresolved textures must fail its job and never reach the cache. +The converter's animation-import ledger re-dup2s the process descriptors and +replays its compacted capture into them, so the per-job files receive the +replayed import output exactly as the single-job log would. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import sys +import tempfile +from pathlib import Path +from typing import Any, Iterator + + +# Real per-job output is a few KiB in practice (import prints plus the glTF +# exporter); the single-job path passes it through unbounded. Multi-job +# markers must stay single-line JSON, so the per-job capture is bounded: a +# job that overflows fails closed instead of risking a truncated guard line. +MAX_JOB_OUTPUT_CAPTURE_BYTES = 1024 * 1024 + + +def _failure_evidence(module: Any, error: BaseException) -> tuple[str | None, str | None]: + """Preserve the converter's sanitized phase classification when present. + + The converter only ever raises ``W3DConversionPhaseError`` with bounded + frozenset values; anything else is reported as type/message alone so no + private payload crosses the process boundary. + """ + + phase_error_type = getattr(module, "W3DConversionPhaseError", None) + if phase_error_type is None or type(error) is not phase_error_type: + return None, None + phase = getattr(error, "failure_phase", None) + kind = getattr(error, "failure_kind", None) + phases = getattr(module, "_W3D_CONVERSION_FAILURE_PHASES", frozenset()) + kinds = getattr(module, "_W3D_CONVERSION_FAILURE_KINDS", frozenset()) + if ( + type(phase) is str + and phase in phases + and type(kind) is str + and kind in kinds + ): + return phase, kind + return None, None + + +def _flush_process_streams() -> None: + """Flush Blender's Python streams before changing process descriptors.""" + + for stream in (sys.stdout, sys.stderr): + try: + stream.flush() + except (AttributeError, OSError, ValueError): + pass + + +@contextlib.contextmanager +def _captured_job_output( + target_fds: tuple[int, ...] = (1, 2), +) -> Iterator[list[Path]]: + """Redirect the process output descriptors into per-job temp files. + + Mirrors the converter ledger's descriptor capture: anything the job (or + the ledger's success replay) writes to stdout/stderr lands in the files + instead of the shared batch output, where per-job attribution would be + impossible. The caller reads and unlinks the files after the block. + """ + + _flush_process_streams() + saved_fds = [os.dup(target_fd) for target_fd in target_fds] + capture_fds: list[int] = [] + capture_paths: list[Path] = [] + redirected = False + try: + try: + for index in range(len(target_fds)): + capture_fd, raw_path = tempfile.mkstemp( + prefix=f"openbfme-w3d-multi-{index}-", suffix=".log" + ) + capture_fds.append(capture_fd) + capture_paths.append(Path(raw_path)) + for target_fd, capture_fd in zip(target_fds, capture_fds): + os.dup2(capture_fd, target_fd) + redirected = True + except BaseException: + for path in capture_paths: + path.unlink(missing_ok=True) + raise + yield capture_paths + finally: + if redirected: + _flush_process_streams() + for target_fd, saved_fd in zip(target_fds, saved_fds): + os.dup2(saved_fd, target_fd) + for file_descriptor in (*saved_fds, *capture_fds): + try: + os.close(file_descriptor) + except OSError: + pass + + +def _read_bounded_job_output(capture_paths: list[Path]) -> str: + """Return the per-job output, laid out like the single-job combined log. + + Fails closed when the capture exceeds the bound: truncating could cut + exactly the line the pipeline's warning-text guards must see. + """ + + chunks: list[bytes] = [] + total = 0 + for path in capture_paths: + data = path.read_bytes() + total += len(data) + chunks.append(data) + if total > MAX_JOB_OUTPUT_CAPTURE_BYTES: + raise RuntimeError( + "W3D multi-job conversion output exceeded the bounded per-job capture" + ) + return b"\n".join(chunks).decode("utf-8", errors="replace") + + +def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Multi-job W3D to GLB (one process)") + parser.add_argument("--plugin-root", type=Path, required=True) + parser.add_argument("--jobs", type=Path, required=True, help="JSON job list") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None, *, converter_module: Any = None) -> int: + # Blender injects its own argv before "--". + if argv is None: + argv = sys.argv[1:] + if "--" in argv: + argv = argv[argv.index("--") + 1 :] + args = _parse_args(argv) + jobs_path = args.jobs.expanduser().resolve() + plugin_root = args.plugin_root.expanduser().resolve() + document = json.loads(jobs_path.read_text(encoding="utf-8")) + if not isinstance(document, dict) or document.get("schema") != "openbfme.w3d-multi-jobs": + raise SystemExit("invalid multi-job document schema") + rows = document.get("jobs") + if not isinstance(rows, list) or not rows: + raise SystemExit("multi-job document has no jobs") + + if converter_module is None: + # Import sibling converter after Blender has set up bpy. + converter_path = Path(__file__).with_name("w3d_to_glb.py") + import importlib.util + + spec = importlib.util.spec_from_file_location( + "_openbfme_w3d_to_glb_multi", converter_path + ) + if spec is None or spec.loader is None: + raise SystemExit("could not load w3d_to_glb.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + else: + module = converter_module + module.initialize_w3d_converter(plugin_root) + + for row in rows: + job_id = str(row["job_id"]) + capture_paths: list[Path] = [] + try: + with _captured_job_output() as capture_paths: + report = module.convert_w3d_job( + model=Path(row["model"]), + asset_kind=str(row["asset_kind"]), + animations=[Path(item) for item in row.get("animations", [])], + required_equipment=list(row.get("required_equipment", [])), + excluded_optional_meshes=list(row.get("excluded_optional_meshes", [])), + proven_root_rigid_bake=bool(row.get("proven_root_rigid_bake", False)), + proven_pivot_only_model=bool(row.get("proven_pivot_only_model", False)), + retail_absent_textures=list(row.get("retail_absent_textures", [])), + output=Path(row["output"]), + ) + output_log = _read_bounded_job_output(capture_paths) + payload: dict[str, Any] = { + "job_id": job_id, + "report": report, + "output_log": output_log, + } + print("OPENBFME_W3D_JOB_OK " + json.dumps(payload, sort_keys=True), flush=True) + except BaseException as exc: + failure_phase, failure_kind = _failure_evidence(module, exc) + payload = { + "job_id": job_id, + "error_type": type(exc).__name__, + "error": str(exc)[:500], + } + if failure_phase is not None and failure_kind is not None: + payload["failure_phase"] = failure_phase + payload["failure_kind"] = failure_kind + print( + "OPENBFME_W3D_JOB_FAIL " + json.dumps(payload, sort_keys=True), + flush=True, + ) + finally: + for capture_path in capture_paths: + capture_path.unlink(missing_ok=True) + print("OPENBFME_W3D_MULTI_DONE " + json.dumps({"jobs": len(rows)}), flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/importer/blender/w3d_to_glb.py b/importer/blender/w3d_to_glb.py index 5ebdb35..c2400d4 100644 --- a/importer/blender/w3d_to_glb.py +++ b/importer/blender/w3d_to_glb.py @@ -583,7 +583,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--animations", type=Path, nargs="*", default=[]) parser.add_argument("--required-equipment", nargs="*", default=[]) parser.add_argument("--excluded-optional-meshes", nargs="*", default=[]) + parser.add_argument("--retail-absent-textures", nargs="*", default=[]) parser.add_argument("--proven-root-rigid-bake", action="store_true") + parser.add_argument("--proven-pivot-only-model", action="store_true") parser.add_argument("--output", type=Path, required=True) return parser.parse_args(argv) @@ -612,12 +614,82 @@ def normalize_optional_mesh_exclusions(value: Any) -> list[str]: return sorted(value) +def normalize_retail_absent_textures(value: Any) -> list[str]: + """Validate scanner-recorded retail-absent texture basenames.""" + + if ( + not isinstance(value, list) + or len(value) > MAX_RETAIL_ABSENT_TEXTURES + or any(not isinstance(basename, str) for basename in value) + ): + raise ValueError( + f"retail-absent textures must be an array of at most " + f"{MAX_RETAIL_ABSENT_TEXTURES} strings" + ) + if len(value) != len(set(value)): + raise ValueError("retail-absent textures contain duplicates") + for basename in value: + if ( + not TEXTURE_BASENAME_PATTERN.fullmatch(basename) + or Path(basename).name != basename + or Path(basename).suffix.casefold() not in TEXTURE_SUFFIXES + ): + raise ValueError( + f"retail-absent texture is not a safe texture basename: {basename!r}" + ) + return sorted(value) + + +def clear_retail_absent_textures(tolerated: list[str]) -> list[str]: + """Unlink generated placeholders for recorded retail-absent textures. + + The pinned importer substitutes a generated color-grid image when a W3D + references a texture that is absent from the staged closure. Retail ships + models whose referenced texture was never published; the visual closure + records each such reference as a ``retail-absent-texture`` exclusion. Only + a generated placeholder whose authored name matches a recorded exclusion + may be unlinked — every other generated image stays for the placeholder + validation to reject. The material keeps all remaining channels; no + substitute texture is invented. + """ + + tolerated_stems = { + Path(basename).stem.casefold() for basename in tolerated + } + unmatched = set(tolerated_stems) + cleared: list[str] = [] + for image in list(getattr(bpy.data, "images", []) or []): + if getattr(image, "source", None) != "GENERATED": + continue + image_name = str(getattr(image, "name", "")) + stem = Path(image_name).stem.casefold() + if stem not in tolerated_stems: + continue + for material in list(getattr(bpy.data, "materials", []) or []): + node_tree = getattr(material, "node_tree", None) + nodes = getattr(node_tree, "nodes", None) + if nodes is None: + continue + for node in list(nodes): + if getattr(node, "image", None) is image: + nodes.remove(node) + bpy.data.images.remove(image) + unmatched.discard(stem) + cleared.append(image_name) + if unmatched: + raise RuntimeError( + "retail-absent texture exclusion did not match a generated placeholder" + ) + return sorted(cleared) + + def validate_asset_kind_request( asset_kind: str, animations: list[Any], required_equipment: list[str], *, proven_root_rigid_bake: bool = False, + proven_pivot_only_model: bool = False, ) -> None: if asset_kind not in {"animated", "hierarchical", "static"}: raise ValueError(f"unsupported W3D asset kind: {asset_kind}") @@ -633,6 +705,14 @@ def validate_asset_kind_request( raise ValueError( "proven root-rigid bake is supported only for hierarchical W3D conversion" ) + if proven_pivot_only_model and asset_kind != "hierarchical": + raise ValueError( + "proven pivot-only model is supported only for hierarchical W3D conversion" + ) + if proven_pivot_only_model and proven_root_rigid_bake: + raise ValueError( + "proven pivot-only model cannot combine with proven root-rigid bake" + ) RENDERABLE_W3D_OBJECT_TYPE = "MESH" @@ -644,6 +724,9 @@ def validate_asset_kind_request( OPAQUE_DESTINATION_BLEND_ENUM = 0 ADDITIVE_ALPHA_EPSILON = 1.0e-8 ADDITIVE_PIXEL_ROUND_TRIP_TOLERANCE = (1.0 / 255.0) + 1.0e-6 +# Byte color attributes quantize through sRGB bytes; the worst-case linear +# round-trip error of one exact conversion is bounded by two byte steps. +ADDITIVE_VERTEX_COLOR_ROUND_TRIP_TOLERANCE = (2.0 / 255.0) + 1.0e-6 SHADER_BOOLEAN_PROPERTY_TYPE = 7 SHADER_BOOLEAN_COMPATIBILITY_PROPERTIES = { "AlphaBlendingEnable": "openbfme_w3d_alpha_blending_enable", @@ -651,6 +734,9 @@ def validate_asset_kind_request( } MAX_OPTIONAL_MESH_EXCLUSIONS = 64 CLEAN_MESH_IDENTIFIER_PATTERN = re.compile(r"^[a-z0-9](?:[a-z0-9_]{0,126}[a-z0-9])?$") +MAX_RETAIL_ABSENT_TEXTURES = 16 +TEXTURE_BASENAME_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +TEXTURE_SUFFIXES = {".bmp", ".dds", ".jpeg", ".jpg", ".png", ".tga"} REDUNDANT_KEYFRAME_WARNING = ( b"Warning: Due to the setting 'Only Insert Needed', " b"1 keyframe(s) have not been inserted." @@ -777,6 +863,7 @@ def __init__( raise FileNotFoundError(self._temp_dir) self._max_bytes = max_bytes self._captured_bytes = 0 + self._suppressed_total = 0 self._destination_fds: tuple[int, int] | None = None self._records: list[tuple[_AnimationImportStreamCapture, ...]] = [] self._finished = False @@ -861,6 +948,16 @@ def capture( if phase_checkpoint is not None: phase_checkpoint.set("animation-output-capture-accounting") + # The exact redundant-keyframe warning class the success replay + # suppresses can dwarf all real output (one fortress build clip emits + # >180 MB of it). Compact it out before accounting so the bound + # measures real output while the exact suppressed count is retained. + for capture in streams: + payload = capture.path.read_bytes() + filtered, suppressed = filter_redundant_keyframe_warning_bytes(payload) + if suppressed: + self._suppressed_total += suppressed + capture.path.write_bytes(filtered) self._captured_bytes += sum(capture.path.stat().st_size for capture in streams) if self._captured_bytes > self._max_bytes: raise RuntimeError( @@ -906,9 +1003,9 @@ def replay_success(self) -> int: if self._finished: raise RuntimeError("animation import output ledger is already closed") captures = list(self._captures()) - suppressed = sum(self._count_suppressed(item.path) for item in captures) for capture in captures: self._replay_filtered(capture) + suppressed = self._suppressed_total self._cleanup() return suppressed @@ -2038,7 +2135,10 @@ def _additive_alpha_pixels(pixels: Iterable[Any]) -> tuple[list[float], dict[str def _verify_additive_pixel_round_trip( - actual_pixels: Iterable[Any], expected_pixels: Iterable[Any] + actual_pixels: Iterable[Any], + expected_pixels: Iterable[Any], + *, + tolerance: float = ADDITIVE_PIXEL_ROUND_TRIP_TOLERANCE, ) -> list[float]: actual = list(actual_pixels) expected = list(expected_pixels) @@ -2058,13 +2158,220 @@ def _verify_additive_pixel_round_trip( or not math.isfinite(target) or channel < 0.0 or channel > 1.0 - or abs(channel - target) > ADDITIVE_PIXEL_ROUND_TRIP_TOLERANCE + or abs(channel - target) > tolerance ): raise RuntimeError("additive material image alpha did not round trip") verified.append(channel) return verified +def _convert_proven_additive_vertex_material( + material: Any, + *, + principled: Any, + base_color: Any, + alpha_input: Any, + links: Any, + phase_checkpoint: _W3DConversionPhaseCheckpoint | None = None, +) -> dict[str, int]: + """Convert a textureless ONE+ONE material whose color source is geometry. + + Some retail additive materials reference no stage-0 texture at all; their + contribution is authored per vertex (the mesh color layer) or as the + material's constant base color. The same exact alpha derivation used for + additive images applies: alpha carries the source intensity and RGB is + normalized so ``normalized_rgb * alpha`` reconstructs the authored + additive contribution. Anything ambiguous (mixed color sources, shared + meshes, conflicting alpha inputs) stays fail-closed. + """ + + _set_optional_phase_checkpoint( + phase_checkpoint, "additive-material-discovery" + ) + meshes = [ + item + for item in list(getattr(bpy.data, "objects", []) or []) + if getattr(item, "type", None) == "MESH" + and any( + getattr(slot, "material", None) is material + for slot in list(getattr(item, "material_slots", []) or []) + ) + ] + if not meshes: + raise RuntimeError("proven additive material has no render mesh") + for item in meshes: + slot_materials = [ + getattr(slot, "material", None) + for slot in list(getattr(item, "material_slots", []) or []) + if getattr(slot, "material", None) is not None + ] + if any(slot_material is not material for slot_material in slot_materials): + raise RuntimeError( + "proven additive vertex material shares its render mesh" + ) + default_value = list(getattr(base_color, "default_value", []) or []) + if len(default_value) != 4: + raise RuntimeError("proven additive material base color is unavailable") + try: + base_channels = [float(value) for value in default_value] + except (TypeError, ValueError) as exc: + raise RuntimeError( + "proven additive material base color is unavailable" + ) from exc + if any(not math.isfinite(value) for value in base_channels): + raise RuntimeError("proven additive material base color is not finite") + + color_attributes = [] + for item in meshes: + attributes = getattr(getattr(item, "data", None), "color_attributes", None) + active = getattr(attributes, "active_color", None) if attributes else None + color_attributes.append(active) + if any(attribute is not None for attribute in color_attributes) and any( + attribute is None for attribute in color_attributes + ): + raise RuntimeError( + "proven additive material has an ambiguous vertex color source" + ) + + report = { + "converted_materials": 1, + "duplicated_images": 0, + "changed_alpha_pixels": 0, + "transparent_pixels": 0, + "visible_pixels": 0, + } + if all(attribute is not None for attribute in color_attributes): + if any(channel > ADDITIVE_ALPHA_EPSILON for channel in base_channels[:3]): + raise RuntimeError( + "proven additive material has an ambiguous color source" + ) + layer_names = {getattr(attribute, "name", "") for attribute in color_attributes} + if len(layer_names) != 1 or not next(iter(layer_names)): + raise RuntimeError( + "proven additive material has an ambiguous vertex color layer" + ) + _set_optional_phase_checkpoint( + phase_checkpoint, "additive-material-alpha-derivation" + ) + for attribute in color_attributes: + data = getattr(attribute, "data", None) + count = len(data) if data is not None else 0 + if count < 1: + raise RuntimeError( + "proven additive material vertex color layer is empty" + ) + buffer = [0.0] * (count * 4) + data.foreach_get("color", buffer) + converted, pixel_report = _additive_alpha_pixels(buffer) + data.foreach_set("color", converted) + round_trip = [0.0] * (count * 4) + data.foreach_get("color", round_trip) + _verify_additive_pixel_round_trip( + round_trip, + converted, + tolerance=ADDITIVE_VERTEX_COLOR_ROUND_TRIP_TOLERANCE, + ) + for key in ( + "changed_alpha_pixels", + "transparent_pixels", + "visible_pixels", + ): + report[key] += pixel_report[key] + try: + base_color.default_value = (1.0, 1.0, 1.0, 1.0) + except (AttributeError, TypeError, ValueError) as exc: + raise RuntimeError( + "proven additive material vertex color source could not be bound" + ) from exc + _set_optional_phase_checkpoint( + phase_checkpoint, "additive-material-image-duplication" + ) + node_tree = getattr(material, "node_tree", None) + nodes = getattr(node_tree, "nodes", None) + if nodes is None: + raise RuntimeError( + "proven additive material has no exportable node graph" + ) + try: + color_node = nodes.new("ShaderNodeVertexColor") + color_node.layer_name = next(iter(layer_names)) + except (AttributeError, RuntimeError, TypeError) as exc: + raise RuntimeError( + "proven additive material vertex color node could not be created" + ) from exc + color_output = _socket_by_name(getattr(color_node, "outputs", None), "Color") + if color_output is None: + raise RuntimeError( + "proven additive material vertex color node has no color output" + ) + try: + links.new(color_output, base_color) + except (AttributeError, RuntimeError, TypeError) as exc: + raise RuntimeError( + "proven additive material vertex color could not be connected" + ) from exc + alpha_output = _socket_by_name(getattr(color_node, "outputs", None), "Alpha") + else: + # Constant-color textureless additive material: derive one exact + # alpha value from the authored base color (black contributes + # nothing and becomes fully transparent; no visibility guard applies + # to a single exact constant). + _set_optional_phase_checkpoint( + phase_checkpoint, "additive-material-alpha-derivation" + ) + red, green, blue = ( + min(1.0, max(0.0, value)) for value in base_channels[:3] + ) + intensity = max(red, green, blue) + if intensity <= ADDITIVE_ALPHA_EPSILON: + normalized = (0.0, 0.0, 0.0) + else: + normalized = (red / intensity, green / intensity, blue / intensity) + try: + base_color.default_value = (*normalized, 1.0) + alpha_input.default_value = intensity + except (AttributeError, TypeError, ValueError) as exc: + raise RuntimeError( + "proven additive material constant color could not be normalized" + ) from exc + report["changed_alpha_pixels"] = int( + abs(intensity - base_channels[3]) > ADDITIVE_ALPHA_EPSILON + ) + report["transparent_pixels"] = int( + intensity < 1.0 - ADDITIVE_ALPHA_EPSILON + ) + report["visible_pixels"] = int(intensity > ADDITIVE_ALPHA_EPSILON) + return report + + _set_optional_phase_checkpoint(phase_checkpoint, "additive-material-alpha-link") + if alpha_output is None: + raise RuntimeError("proven additive material image has no alpha output") + alpha_input_identity = _runtime_identity(alpha_input) + incoming_alpha = [ + link + for link in list(links) + if _runtime_identity(getattr(link, "to_socket", None)) == alpha_input_identity + ] + if incoming_alpha: + if len(incoming_alpha) != 1 or ( + not _same_runtime_identity( + getattr(incoming_alpha[0], "from_node", None), color_node + ) + or not _same_runtime_identity( + getattr(incoming_alpha[0], "from_socket", None), alpha_output + ) + ): + raise RuntimeError("proven additive material has an ambiguous alpha input") + else: + try: + links.new(alpha_output, alpha_input) + except (AttributeError, RuntimeError, TypeError) as exc: + raise RuntimeError( + "additive material alpha could not be connected" + ) from exc + return report + + def _convert_proven_additive_material( material: Any, *, @@ -2113,6 +2420,15 @@ def _convert_proven_additive_material( list(direct_color_nodes.values()) if direct_color_nodes else image_nodes ) if len(candidates) != 1: + if not image_nodes: + return _convert_proven_additive_vertex_material( + material, + principled=principled, + base_color=base_color, + alpha_input=alpha_input, + links=links, + phase_checkpoint=phase_checkpoint, + ) raise RuntimeError("proven additive material has an ambiguous color image") image_node = candidates[0] source_image = image_node.image @@ -2651,12 +2967,16 @@ def bake_proven_root_rigid_hierarchy( mesh_objects: list[Any], object_collection: Any, ) -> dict[str, Any]: - """Bake one scanner-proven pivot-zero carrier into rigid scene meshes. + """Bake one scanner-proven rigid carrier into rigid scene meshes. OpenSAGE deliberately omits the source root pivot. For a model whose every render reference is planner-proven to target that pivot, the importer emits - one empty armature carrier with rigid mesh children. This opt-in path removes - only that exact carrier shape while proving that world transforms survive. + one empty armature carrier with rigid mesh children. A static multi-pivot + hierarchy whose render meshes are all rigidly attached (bone-parented or + carrier-parented, never skinned) is the same carrier shape with more rest + pivots: the pivots are rigid rest transforms because nothing can animate + them. This opt-in path removes only those exact carrier shapes while + proving that world transforms survive. """ if requested is not True: @@ -2671,8 +2991,11 @@ def bake_proven_root_rigid_hierarchy( raise RuntimeError( "proven root-rigid bake requires exactly one armature carrier" ) - if len(list(getattr(getattr(rig, "data", None), "bones", []) or [])) != 0: - raise RuntimeError("proven root-rigid bake carrier is not empty") + # Pivots may only be rigid rest transforms when nothing animates them: no + # actions anywhere in the scene (proven below) and no bone constraints. + for pose_bone in list(getattr(getattr(rig, "pose", None), "bones", []) or []): + if list(getattr(pose_bone, "constraints", []) or []): + raise RuntimeError("proven root-rigid carrier bone has constraints") assert_non_animated_scene_has_no_actions(asset_kind) if ( getattr(rig, "parent", None) is not None @@ -2703,17 +3026,29 @@ def bake_proven_root_rigid_hierarchy( world_transforms: list[tuple[Any, Any]] = [] for item in mesh_objects: - if ( - getattr(item, "type", None) != "MESH" - or getattr(item, "parent", None) is not rig - or str(getattr(item, "parent_type", "")) != "ARMATURE" - or str(getattr(item, "parent_bone", "")) - ): + if getattr(item, "type", None) != "MESH": raise RuntimeError( "proven root-rigid render mesh is not rigidly parented to the carrier" ) - if list(getattr(item, "vertex_groups", []) or []) or list( - getattr(item, "modifiers", []) or [] + parent = getattr(item, "parent", None) + parent_type = str(getattr(item, "parent_type", "")) + if parent is None: + if parent_type != "OBJECT": + raise RuntimeError( + "proven root-rigid render mesh is not rigidly parented to the carrier" + ) + elif parent is not rig or parent_type not in {"ARMATURE", "OBJECT", "BONE"}: + raise RuntimeError( + "proven root-rigid render mesh is not rigidly parented to the carrier" + ) + if list(getattr(item, "vertex_groups", []) or []): + raise RuntimeError( + "proven root-rigid render mesh has ambiguous deformation state" + ) + inert_modifiers = list(getattr(item, "modifiers", []) or []) + if any( + str(getattr(modifier, "type", "")) != "ARMATURE" + for modifier in inert_modifiers ): raise RuntimeError( "proven root-rigid render mesh has ambiguous deformation state" @@ -2728,6 +3063,10 @@ def bake_proven_root_rigid_hierarchy( for item, world in world_transforms: try: + # Armature modifiers are deformation-inert without vertex weights + # (proven above) and dangle once the carrier is removed. + for modifier in list(getattr(item, "modifiers", []) or []): + item.modifiers.remove(modifier) item.parent = None item.parent_type = "OBJECT" item.parent_bone = "" @@ -2908,6 +3247,79 @@ def find_single_rig() -> bpy.types.Object: return rigs[0] +def find_model_rig(asset_kind: str) -> Any: + """Resolve the model rig; animated composite carriers may be rigless. + + A rigless model is legitimate only for animated conversion, where every + requested clip must key its own auxiliary rig (composite citadel models + ship static base meshes while their clips target sibling hierarchies). + Hierarchical and static conversions keep their exact single/zero rig + contracts. + """ + + if asset_kind == "static": + return find_static_rig() + rigs = [item for item in bpy.data.objects if item.type == "ARMATURE"] + if asset_kind == "animated" and not rigs: + return None + if len(rigs) != 1: + raise RuntimeError( + f"expected one armature after model import, found {len(rigs)}" + ) + return rigs[0] + + +def _scene_armature_objects() -> list[Any]: + return [ + item + for item in list(getattr(bpy.data, "objects", []) or []) + if getattr(item, "type", None) == "ARMATURE" + ] + + +def _owned_active_actions(candidate: Any) -> list[Any]: + owned: list[Any] = [] + for owner in (candidate, getattr(candidate, "data", None)): + animation_data = getattr(owner, "animation_data", None) + active = getattr(animation_data, "action", None) + if active is not None: + owned.append(active) + return owned + + +def find_animation_owner_rig(model_rig: Any, created_actions: list[Any]) -> Any: + """Return the single rig owning every action one clip import created. + + Cross-hierarchy clips (mounted rigs, composite citadel siblings) key the + auxiliary armature the pinned importer creates for their own hierarchy; + same-hierarchy clips key the model rig. The owner must be unique or the + capture that follows could silently attribute curves to the wrong rig. + """ + + if not created_actions: + raise RuntimeError("W3D animation did not create an owned keyed action") + created_identities = {_runtime_identity(action) for action in created_actions} + candidates: list[tuple[Any, set[Any]]] = [] + rigs = _scene_armature_objects() + if model_rig is not None and all(item is not model_rig for item in rigs): + rigs.append(model_rig) + for candidate in rigs: + owned_identities = { + _runtime_identity(action) + for action in _owned_active_actions(candidate) + } + if owned_identities & created_identities: + candidates.append((candidate, owned_identities)) + if not candidates: + raise RuntimeError("W3D animation did not create an owned keyed action") + if len(candidates) != 1: + raise RuntimeError("W3D animation created actions across ambiguous owner rigs") + candidate, owned_identities = candidates[0] + if not created_identities.issubset(owned_identities): + raise RuntimeError("W3D animation created actions outside its proven owner set") + return candidate + + def find_static_rig() -> Any: """Reject skeletal static imports instead of silently baking an arbitrary pose.""" @@ -3135,33 +3547,54 @@ def capture_split_w3d_animation_actions(rig: Any, actions: Iterable[Any]) -> lis def prepare_w3d_animation_nla_tracks( rig: Any, action_shapes: list[dict[str, Any]] ) -> int: - """Bind each logical W3D transform action to one named NLA export track.""" - - detach_actions(rig) - animation_data_create = getattr(rig, "animation_data_create", None) - if callable(animation_data_create): - animation_data_create() - animation_data = getattr(rig, "animation_data", None) - tracks = getattr(animation_data, "nla_tracks", None) - if tracks is None: - raise RuntimeError("W3D rig has no NLA track collection") - while len(tracks): - tracks.remove(tracks[0]) - created = 0 - for shape in action_shapes: - action = shape.get("object_action") - if action is None or shape["public"]["transform_curve_count"] < 1: + """Bind each logical W3D transform action to one named NLA export track. + + Each clip is tracked on the rig that actually owns its actions: the model + rig for same-hierarchy clips, the auxiliary armature for cross-hierarchy + clips. Shapes without an owner record belong to the model rig. + """ + + owner_rigs: list[Any] = [] + seen_identities: set[Any] = set() + for candidate in [rig, *(shape.get("owner_rig") for shape in action_shapes)]: + if candidate is None: continue - track = tracks.new() - track.name = shape["public"]["name"] - frame_range = getattr(action, "frame_range", (0.0, 0.0)) - raw_start = float(frame_range[0]) - start = int(round(raw_start)) - if abs(raw_start - start) > 1.0e-6: - raise RuntimeError("W3D action has a fractional NLA start frame") - strip = track.strips.new(shape["public"]["name"], start, action) - strip.name = shape["public"]["name"] - created += 1 + identity = _runtime_identity(candidate) + if identity not in seen_identities: + seen_identities.add(identity) + owner_rigs.append(candidate) + created = 0 + for owner in owner_rigs: + detach_actions(owner) + animation_data_create = getattr(owner, "animation_data_create", None) + if callable(animation_data_create): + animation_data_create() + animation_data = getattr(owner, "animation_data", None) + tracks = getattr(animation_data, "nla_tracks", None) + if tracks is None: + raise RuntimeError("W3D rig has no NLA track collection") + while len(tracks): + tracks.remove(tracks[0]) + owner_identity = _runtime_identity(owner) + for shape in action_shapes: + shape_owner = shape.get("owner_rig", rig) + if shape_owner is None or _runtime_identity(shape_owner) != ( + owner_identity + ): + continue + action = shape.get("object_action") + if action is None or shape["public"]["transform_curve_count"] < 1: + continue + track = tracks.new() + track.name = shape["public"]["name"] + frame_range = getattr(action, "frame_range", (0.0, 0.0)) + raw_start = float(frame_range[0]) + start = int(round(raw_start)) + if abs(raw_start - start) > 1.0e-6: + raise RuntimeError("W3D action has a fractional NLA start frame") + strip = track.strips.new(shape["public"]["name"], start, action) + strip.name = shape["public"]["name"] + created += 1 expected = sum( 1 for shape in action_shapes if shape["public"]["transform_curve_count"] > 0 ) @@ -3360,9 +3793,19 @@ def restore_duplicate_logical_animations( def validate_split_animation_glb( - output: Path, expected_names: Iterable[str] + output: Path, + expected_names: Iterable[str], + *, + require_skins: bool = True, + require_skeletal_mesh: bool = True, ) -> dict[str, int]: - """Require the exact emitted glTF animation set and skeletal geometry.""" + """Require the exact emitted glTF animation set and skeletal geometry. + + ``require_skins`` is the scene-proven contract that skinned meshes had to + survive the export. Proven rigid animated models carry no skinned meshes; + their bone- or armature-parented render meshes are skeletal content even + though the GLB has no skins array. + """ expected = [clean_name(name) for name in expected_names] if any(not name for name in expected) or len(expected) != len(set(expected)): @@ -3559,8 +4002,12 @@ def validate_split_animation_glb( skins = document.get("skins") nodes = document.get("nodes") - if not isinstance(skins, list) or not skins: + if require_skins and (not isinstance(skins, list) or not skins): raise RuntimeError("split-animation GLB has no skeletal skin") + if skins is None: + skins = [] + if not isinstance(skins, list): + raise RuntimeError("split-animation GLB has an invalid skeletal skin array") if not isinstance(nodes, list) or not nodes: raise RuntimeError("split-animation GLB has no nodes") joint_nodes: set[int] = set() @@ -3592,7 +4039,37 @@ def validate_split_animation_glb( raise RuntimeError("split-animation GLB node has multiple parents") parents[child] = parent_index + # Proven rigid animated models have no skins; their render meshes are + # still skeletal content when they hang off a bone (under a joint) or off + # the armature itself. Armature-parented meshes are a fallback so exact + # counts of skinned or bone-parented meshes never change. With no joints + # and no exported channels (an empty single-pivot carrier whose clip only + # keys object visibility), the armature node is the mesh's non-mesh + # parent itself. + armature_nodes: set[int] = set() + if not require_skins: + for child in joint_nodes: + parent = parents.get(child) + if parent is not None and parent not in joint_nodes: + armature_nodes.add(parent) + for animation in animations: + channels = animation.get("channels") if isinstance(animation, dict) else [] + for channel in channels if isinstance(channels, list) else []: + if not isinstance(channel, dict): + continue + target = channel.get("target") + if not isinstance(target, dict): + continue + target_node = target.get("node") + if isinstance(target_node, int): + parent = parents.get(target_node) + if parent is not None: + armature_nodes.add(parent) + for node_index, node in enumerate(nodes): + if isinstance(node, dict) and not isinstance(node.get("mesh"), int): + armature_nodes.add(node_index) skeletal_mesh_count = 0 + armature_parented_mesh_count = 0 for node_index, node in enumerate(nodes): if not isinstance(node.get("mesh"), int): continue @@ -3602,13 +4079,24 @@ def validate_split_animation_glb( continue seen: set[int] = set() parent = parents.get(node_index) + bone_parented = False + armature_parented = False while parent is not None and parent not in seen: if parent in joint_nodes: - skeletal_mesh_count += 1 + bone_parented = True + break + if parent in armature_nodes: + armature_parented = True break seen.add(parent) parent = parents.get(parent) - if skeletal_mesh_count < 1: + if bone_parented: + skeletal_mesh_count += 1 + elif armature_parented: + armature_parented_mesh_count += 1 + if skeletal_mesh_count == 0: + skeletal_mesh_count = armature_parented_mesh_count + if require_skeletal_mesh and skeletal_mesh_count < 1: raise RuntimeError( "split-animation GLB has no skinned or bone-parented mesh node" ) @@ -3717,6 +4205,12 @@ def initialize_w3d_converter(plugin_root: Path) -> None: sys.path.insert(0, plugin_root_text) import io_mesh_w3d # type: ignore + # The third-party add-on updater writes status beside its source by + # default. Keep the hash-pinned plugin immutable and avoid Windows path + # length failures by redirecting that disposable state to the OS temp + # directory before registration. + updater = io_mesh_w3d.addon_updater_ops.updater + updater.stage_path = tempfile.mkdtemp(prefix="openbfme-w3d-updater-") io_mesh_w3d.register() install_shader_material_compatibility_shim() _INITIALIZED_W3D_PLUGIN_ROOT = plugin_root @@ -3821,6 +4315,8 @@ def _convert_w3d_job_impl( required_equipment: list[str], excluded_optional_meshes: list[str], proven_root_rigid_bake: bool, + proven_pivot_only_model: bool = False, + retail_absent_textures: list[str] | None = None, output: Path, animation_output_ledger: AnimationImportOutputLedger, phase_checkpoint: _W3DConversionPhaseCheckpoint, @@ -3833,6 +4329,10 @@ def _convert_w3d_job_impl( required_equipment=required_equipment, excluded_optional_meshes=excluded_optional_meshes, proven_root_rigid_bake=proven_root_rigid_bake, + proven_pivot_only_model=proven_pivot_only_model, + retail_absent_textures=normalize_retail_absent_textures( + retail_absent_textures or [] + ), ) model = model.expanduser().resolve() output = output.expanduser().resolve() @@ -3875,24 +4375,27 @@ def _convert_w3d_job_impl( args.animations, args.required_equipment, proven_root_rigid_bake=args.proven_root_rigid_bake, + proven_pivot_only_model=args.proven_pivot_only_model, ) phase_checkpoint.set("rig-validation") phase_checkpoint.set("rig-resolution") - rig = find_static_rig() if args.asset_kind == "static" else find_single_rig() + rig = find_model_rig(args.asset_kind) phase_checkpoint.set("action-validation") assert_non_animated_scene_has_no_actions(args.asset_kind) phase_checkpoint.set("geometry-validation") filtered_geometry = remove_non_render_geometry() model_mesh_objects = [item for item in bpy.data.objects if item.type == "MESH"] - if not model_mesh_objects: + if not model_mesh_objects and not args.proven_pivot_only_model: raise RuntimeError("W3D model import created no meshes") phase_checkpoint.set("skin-validation") - if ( - rig is not None - and len(getattr(rig.data, "bones", []) or []) < 1 - and not args.proven_root_rigid_bake - ): - raise RuntimeError("skeletal W3D import has an empty hierarchy") + # Recorded retail-absent textures are unlinked before material passes see + # the graph, so their placeholders never leak into additive or opaque + # material conversion. Only scanner-recorded exclusions are tolerated. + retail_absent_textures_cleared = ( + clear_retail_absent_textures(args.retail_absent_textures) + if args.retail_absent_textures + else [] + ) # Preserve the visual contribution of source-proven additive W3D textures # before the render payload is fingerprinted. Unproven materials are never # modified by this pass. @@ -3912,7 +4415,7 @@ def _convert_w3d_job_impl( ) model_mesh_objects = [item for item in bpy.data.objects if item.type == "MESH"] model_mesh_count = len(model_mesh_objects) - if model_mesh_count < 1: + if model_mesh_count < 1 and not args.proven_pivot_only_model: raise RuntimeError("W3D model import retained no meshes") root_rigid_bake = { "requested": False, @@ -3947,6 +4450,20 @@ def _convert_w3d_job_impl( rig, phase_checkpoint=phase_checkpoint, ) + # An empty armature is only degenerate when real bone-deformed content + # needs pivots. Rigid render meshes on an empty carrier (retail's + # single-pivot models with object-level visibility clips) are a + # legitimate retail shape. The check runs after the mesh inventory (it + # needs the skinned flags), so re-anchor the phase to skin validation + # instead of leaving the inventory's last checkpoint as the evidence. + phase_checkpoint.set("skin-validation") + if ( + rig is not None + and len(getattr(rig.data, "bones", []) or []) < 1 + and not args.proven_root_rigid_bake + and any(item["skinned"] for item in mesh_inventory) + ): + raise RuntimeError("skeletal W3D import has an empty hierarchy") phase_checkpoint.set("render-proof") render_geometry_proof = capture_render_geometry_proof(model_mesh_objects) render_attachment_proof = ( @@ -3973,6 +4490,7 @@ def _convert_w3d_job_impl( preserved_visibility_channel_count = 0 preserved_visibility_key_count = 0 visibility_only_sidecar_animation_count = 0 + discarded_embedded_model_action_count = 0 phase_checkpoint.set("animation-import") if embedded_model_animation: if rig is None: @@ -3980,6 +4498,7 @@ def _convert_w3d_job_impl( imported_actions, action_shape = capture_w3d_animation_actions( rig, list(bpy.data.actions), resolved_animations[0].stem ) + action_shape["owner_rig"] = rig animation_action_shapes.append(action_shape) logical_animation_count = 1 split_action_animation_count = int( @@ -3987,10 +4506,25 @@ def _convert_w3d_job_impl( and action_shape["public"]["armature_action_count"] == 1 ) elif rig is not None: - if list(bpy.data.actions): - raise RuntimeError( - "W3D model import contains unexpected embedded animation actions" - ) + stray_embedded_actions = list(bpy.data.actions) + if stray_embedded_actions: + if not resolved_animations: + raise RuntimeError( + "W3D model import contains unexpected embedded animation actions" + ) + # RotWK 2.01 models embed a redundant one-channel pose clip + # beside the externally authored state clips (kbangwgn_a.w3d + # embeds KBANGWGN_ASKL.KBANGWGN_A while retail binds the _ABLD + # buildup clip). The attached external clips are the authored + # presentation this job declares; the embedded pose actions are + # removed here with explicit report evidence — never silently. + discarded_embedded_model_action_count = len(stray_embedded_actions) + for stray_action in stray_embedded_actions: + bpy.data.actions.remove(stray_action) + if list(bpy.data.actions): + raise RuntimeError( + "embedded W3D pose actions were not fully discarded" + ) detach_actions(rig) for source in resolved_animations: if embedded_model_animation: @@ -4014,15 +4548,20 @@ def _convert_w3d_job_impl( created = sorted(after - before, key=lambda item: item.name.casefold()) if not created: active_actions = [] - for owner in (rig, rig.data): - animation_data = getattr(owner, "animation_data", None) - active = getattr(animation_data, "action", None) - if active is not None and active not in before: - active_actions.append(active) + for candidate in _scene_armature_objects(): + for active in _owned_active_actions(candidate): + if active not in before: + active_actions.append(active) created = active_actions + # Cross-hierarchy clips key the auxiliary armature their own hierarchy + # creates; same-hierarchy clips key the model rig. Capture and detach + # on the true owner so the next clip can never reuse and merge into + # this clip's still-assigned action. + owner_rig = find_animation_owner_rig(rig, created) captured, action_shape = capture_w3d_animation_actions( - rig, created, source.stem + owner_rig, created, source.stem ) + action_shape["owner_rig"] = owner_rig imported_actions.extend(captured) animation_action_shapes.append(action_shape) logical_animation_count += 1 @@ -4030,7 +4569,7 @@ def _convert_w3d_job_impl( action_shape["public"]["object_action_count"] == 1 and action_shape["public"]["armature_action_count"] == 1 ) - detach_actions(rig) + detach_actions(owner_rig) phase_checkpoint.set("scene-validation") phase_checkpoint.set("post-animation-validation") @@ -4119,6 +4658,16 @@ def _convert_w3d_job_impl( for shape in animation_action_shapes if shape["public"]["transform_curve_count"] > 0 ], + require_skins=skinned_meshes > 0, + require_skeletal_mesh=( + skinned_meshes > 0 + or ( + rig is not None + and any( + getattr(item, "parent", None) is rig for item in mesh_objects + ) + ) + ), ) if ( action_shape_export["visibility_channels"] @@ -4141,6 +4690,7 @@ def _convert_w3d_job_impl( "mesh_inventory": mesh_inventory, "required_equipment": sorted(set(args.required_equipment)), "equipment": equipment, + "retail_absent_textures_cleared": retail_absent_textures_cleared, "animations": logical_animation_count, "animation_curves": animation_curve_count, "animation_keys": animation_key_count, @@ -4167,6 +4717,9 @@ def _convert_w3d_job_impl( "embedded_model_action_count": ( len(imported_actions) if embedded_model_animation else 0 ), + "discarded_embedded_model_action_count": ( + discarded_embedded_model_action_count + ), "embedded_exported_animation_count": embedded_export["animations"], "embedded_exported_channel_count": embedded_export["channels"], "embedded_exported_sampler_count": embedded_export["samplers"], @@ -4213,6 +4766,8 @@ def convert_w3d_job( excluded_optional_meshes: list[str], proven_root_rigid_bake: bool, output: Path, + proven_pivot_only_model: bool = False, + retail_absent_textures: list[str] | None = None, ) -> dict[str, Any]: """Convert one job while retaining raw animation-import output on failure.""" @@ -4228,6 +4783,8 @@ def convert_w3d_job( required_equipment=required_equipment, excluded_optional_meshes=excluded_optional_meshes, proven_root_rigid_bake=proven_root_rigid_bake, + proven_pivot_only_model=proven_pivot_only_model, + retail_absent_textures=retail_absent_textures, output=output, animation_output_ledger=animation_output_ledger, phase_checkpoint=phase_checkpoint, @@ -4263,6 +4820,8 @@ def main() -> None: required_equipment=args.required_equipment, excluded_optional_meshes=args.excluded_optional_meshes, proven_root_rigid_bake=args.proven_root_rigid_bake, + proven_pivot_only_model=args.proven_pivot_only_model, + retail_absent_textures=args.retail_absent_textures, output=args.output, ) print("OPENBFME_W3D_OK " + json.dumps(report, sort_keys=True)) diff --git a/importer/openbfme_importer/big.py b/importer/openbfme_importer/big.py index 22a934e..e885750 100644 --- a/importer/openbfme_importer/big.py +++ b/importer/openbfme_importer/big.py @@ -245,24 +245,31 @@ def extract( raise FileExistsError( f"cached extraction size mismatch; use --force: {target}" ) + # Default warm path: hash the cached file only (one pass). + # Full archive+file dual re-hash was ~2× slower for no-ops. + # Set OPENBFME_EXTRACT_VERIFY=full to re-read archive bytes. + verify_mode = os.environ.get( + "OPENBFME_EXTRACT_VERIFY", "cached" + ).strip().casefold() cached_digest = sha256_file(target) - source.seek(entry.offset) - source_digest = hashlib.sha256() - remaining = entry.size - while remaining: - chunk = source.read(min(COPY_CHUNK, remaining)) - if not chunk: - raise BigFormatError( - f"unexpected EOF verifying cached {entry.name!r}" + if verify_mode in {"full", "dual", "archive"}: + source.seek(entry.offset) + source_digest = hashlib.sha256() + remaining = entry.size + while remaining: + chunk = source.read(min(COPY_CHUNK, remaining)) + if not chunk: + raise BigFormatError( + f"unexpected EOF verifying cached {entry.name!r}" + ) + source_digest.update(chunk) + remaining -= len(chunk) + current_digest = source_digest.hexdigest() + if cached_digest != current_digest: + raise FileExistsError( + f"cached extraction hash mismatch; use --force: {target}" ) - source_digest.update(chunk) - remaining -= len(chunk) - current_digest = source_digest.hexdigest() - if cached_digest != current_digest: - raise FileExistsError( - f"cached extraction hash mismatch; use --force: {target}" - ) - results.append(ExtractedEntry(entry, target, current_digest)) + results.append(ExtractedEntry(entry, target, cached_digest)) continue temp = target.with_name(target.name + ".openbfme-part") diff --git a/importer/openbfme_importer/bootstrap.py b/importer/openbfme_importer/bootstrap.py index 1246338..fb10555 100644 --- a/importer/openbfme_importer/bootstrap.py +++ b/importer/openbfme_importer/bootstrap.py @@ -15,6 +15,7 @@ import sys import tempfile import urllib.request +import urllib.parse import zipfile from typing import Any @@ -32,7 +33,23 @@ PLUGIN_REPOSITORY = "https://github.com/OpenSAGE/OpenSAGE.BlenderPlugin.git" PLUGIN_COMMIT = "2de84023cb632a79a853b2a52f97c8002ed85142" PLUGIN_SUBMODULE_COMMIT = "981aa2984117a1c686b7fa40d086794ce1c7665e" +PLUGIN_ARCHIVE_URL = ( + "https://github.com/OpenSAGE/OpenSAGE.BlenderPlugin/archive/" + f"{PLUGIN_COMMIT}.zip" +) +PLUGIN_ARCHIVE_SHA256 = "0e4ff63e8a9e9c04c4fa94eb232893624a47d39fe89cfbf34082353a06567c29" +PLUGIN_SUBMODULE_ARCHIVE_URL = ( + "https://github.com/CGCookie/blender-addon-updater/archive/" + f"{PLUGIN_SUBMODULE_COMMIT}.zip" +) +PLUGIN_SUBMODULE_ARCHIVE_SHA256 = "5a78744e3eb3bfa33e2d007b345697eccf7f9920f73fa828212da8e7691f4377" +PLUGIN_TREE_SHA256 = "d64e49a9daba7dbec5f7d6bee4e947208f763b97de038e752c8213cf6a13bd3f" FFMPEG_VERSION = "8.1.1" +FFMPEG_ARCHIVE_URL = ( + "https://github.com/GyanD/codexffmpeg/releases/download/8.1.1/" + "ffmpeg-8.1.1-essentials_build.zip" +) +FFMPEG_ARCHIVE_SHA256 = "6f58ce889f59c311410f7d2b18895b33c03456463486f3b1ebc93d97a0f54541" FFMPEG_EXE_SHA256 = "228d7a8556258de907fdb55f36850078ebc7680b84ec30d84ea02e99bec1d1eb" FFPROBE_EXE_SHA256 = "0fde260f5abd35c9cafd96f594cc76365a780c1b73a90e35b6a3409ea1db1bf0" PILLOW_TREE_SHA256 = "18c02c91b31a5b2619eb1542144f0ef1f7ac4065eab7c5924f2640b3010fd7b0" @@ -42,6 +59,7 @@ DEFUSEDXML_TREE_SHA256 = "4a5bc129bad371fd21f6bb07621d2d331a1d2b192fef9b2bf78656b928c7738d" PYTHON_VERSION = "3.12.10" PYTHON_LAUNCHER_SHA256 = "0b471133e110cfb53a061cad528ce8e517d7b9ac41a0a396c39ad795a487fc14" +PYTHON_BASE_LAUNCHER_SHA256 = "4d6f5f81a4bca11191c4c7c6b43632694d0a4ce74e068619d8fdc161d469859a" PYTHON_BASE_DLL_SHA256 = "9a0e3435aaa680d868150f87ab3e388ad2eebc22f87e036155c7b4eda8cd2120" PYTHON_RUNTIME_TREE_SHA256 = "98348e31da2e14c684372bf02fee52b71984d28d8a91b82dbe0fe9aa2f6561d7" PYTHON_RUNTIME_MAX_FILES = 20_000 @@ -61,6 +79,48 @@ } +def _download_file(url: str, destination: Path, *, max_bytes: int) -> None: + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https" or parsed.hostname not in { + "github.com", + "codeload.github.com", + "download.blender.org", + "objects.githubusercontent.com", + "release-assets.githubusercontent.com", + }: + raise RuntimeError(f"tool download URL is not approved: {url}") + request = urllib.request.Request( + url, + headers={"User-Agent": "OpenBFME-Importer/1.0"}, + ) + temporary = destination.with_suffix(destination.suffix + ".part") + total = 0 + try: + with urllib.request.urlopen(request, timeout=60) as response: + final = urllib.parse.urlparse(response.geturl()) + if final.scheme != "https" or final.hostname not in { + "github.com", + "codeload.github.com", + "download.blender.org", + "objects.githubusercontent.com", + "release-assets.githubusercontent.com", + }: + raise RuntimeError("tool download redirected to an unapproved host") + declared = response.headers.get("Content-Length") + if declared and int(declared) > max_bytes: + raise RuntimeError("tool download exceeds its size bound") + with temporary.open("xb") as output: + while chunk := response.read(1024 * 1024): + total += len(chunk) + if total > max_bytes: + raise RuntimeError("tool download exceeds its size bound") + output.write(chunk) + os.replace(temporary, destination) + finally: + if temporary.exists(): + temporary.unlink() + + def _reject_tree_links(root: Path, label: str) -> None: requested_root = Path(root).expanduser() if _is_link_or_junction(requested_root): @@ -349,14 +409,21 @@ def prepare_opensage_plugin_checkout( state_root: Path, plugin: Path | None = None, ) -> dict[str, str]: - """Recover caches only in the state root's pinned plugin, then attest Git.""" + """Recover caches, then attest the pinned Git or archive source tree.""" resolved_state_root = Path(state_root).expanduser().resolve() pinned = resolved_state_root / "tools" / "OpenSAGE.BlenderPlugin" selected = Path(plugin or pinned).expanduser().resolve(strict=True) if selected == pinned: _purge_python_caches(selected, "OpenSAGE W3D plugin") - return _attest_opensage_plugin_checkout(selected) + if (selected / ".git").exists(): + return _attest_opensage_plugin_checkout(selected) + _attest_opensage_plugin_portable(selected) + return { + "commit": PLUGIN_COMMIT, + "submodule_commit": PLUGIN_SUBMODULE_COMMIT, + "tree_sha256": PLUGIN_TREE_SHA256, + } def python_runtime_attestation() -> dict[str, Any]: @@ -441,6 +508,18 @@ def python_runtime_attestation() -> dict[str, Any]: } +def python_runtime_identity_is_pinned(report: dict[str, Any]) -> bool: + """Accept the pinned venv stub or standalone base launcher.""" + + return ( + report.get("version") == PYTHON_VERSION + and report.get("launcher_sha256") + in {PYTHON_LAUNCHER_SHA256, PYTHON_BASE_LAUNCHER_SHA256} + and report.get("base_dll_sha256") == PYTHON_BASE_DLL_SHA256 + and report.get("tree_sha256") == PYTHON_RUNTIME_TREE_SHA256 + ) + + def _run(command: list[str], *, cwd: Path | None = None) -> str: try: result = subprocess.run( @@ -479,7 +558,7 @@ def _download_blender(tools_root: Path) -> tuple[Path, str]: else: temporary = zip_path.with_suffix(".zip.downloading") temporary.unlink(missing_ok=True) - urllib.request.urlretrieve(BLENDER_URL, temporary) + _download_file(BLENDER_URL, temporary, max_bytes=1024 * 1024 * 1024) _require_hash(temporary, BLENDER_ZIP_SHA256, "Blender archive") os.replace(temporary, zip_path) with tempfile.TemporaryDirectory(dir=tools_root, prefix="blender-extract-") as raw: @@ -511,20 +590,77 @@ def _download_blender(tools_root: Path) -> tuple[Path, str]: def _checkout_plugin(tools_root: Path) -> Path: destination = tools_root / "OpenSAGE.BlenderPlugin" git = shutil.which("git") - if not git: - raise FileNotFoundError("git is required to provision the OpenSAGE W3D plugin") if (destination / ".git").exists(): prepare_opensage_plugin_checkout(tools_root.parent, destination) return destination - if not (destination / ".git").exists(): - _run([git, "clone", "--no-checkout", PLUGIN_REPOSITORY, str(destination)]) - _run([git, "fetch", "--depth", "1", "origin", PLUGIN_COMMIT], cwd=destination) - _run([git, "checkout", "--detach", PLUGIN_COMMIT], cwd=destination) - _run([git, "submodule", "update", "--init", "--depth", "1"], cwd=destination) - _attest_opensage_plugin_checkout(destination) + if destination.is_dir(): + _attest_opensage_plugin_portable(destination) + return destination + with tempfile.TemporaryDirectory(prefix="opensage-", dir=tools_root) as temporary: + staging = Path(temporary) + plugin_zip = staging / "plugin.zip" + updater_zip = staging / "updater.zip" + _download_file(PLUGIN_ARCHIVE_URL, plugin_zip, max_bytes=128 * 1024 * 1024) + _download_file( + PLUGIN_SUBMODULE_ARCHIVE_URL, + updater_zip, + max_bytes=128 * 1024 * 1024, + ) + _require_hash(plugin_zip, PLUGIN_ARCHIVE_SHA256, "OpenSAGE plugin archive") + _require_hash( + updater_zip, + PLUGIN_SUBMODULE_ARCHIVE_SHA256, + "OpenSAGE plugin updater archive", + ) + plugin_extract = staging / "plugin" + updater_extract = staging / "updater" + _extract_safe_zip(plugin_zip, plugin_extract) + _extract_safe_zip(updater_zip, updater_extract) + plugin_roots = [item for item in plugin_extract.iterdir() if item.is_dir()] + updater_roots = [item for item in updater_extract.iterdir() if item.is_dir()] + if len(plugin_roots) != 1 or len(updater_roots) != 1: + raise RuntimeError("OpenSAGE source archives have an unexpected layout") + assembled = staging / "assembled" + shutil.move(str(plugin_roots[0]), assembled) + updater_target = assembled / "io_mesh_w3d" / "blender_addon_updater" + if updater_target.exists(): + if updater_target.is_dir(): + shutil.rmtree(updater_target) + else: + updater_target.unlink() + shutil.move(str(updater_roots[0]), updater_target) + _attest_opensage_plugin_portable(assembled) + os.replace(assembled, destination) return destination +def _extract_safe_zip(archive_path: Path, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=False) + canonical = destination.resolve() + with zipfile.ZipFile(archive_path) as archive: + for member in archive.infolist(): + parts = safe_relative_parts(member.filename.rstrip("/")) + target = destination.joinpath(*parts).resolve() + try: + target.relative_to(canonical) + except ValueError as exc: + raise RuntimeError(f"ZIP entry escapes destination: {member.filename}") from exc + if member.external_attr and ((member.external_attr >> 16) & 0xF000) == 0xA000: + raise RuntimeError(f"ZIP contains a symbolic link: {member.filename}") + archive.extractall(destination) + + +def _attest_opensage_plugin_portable(checkout: Path) -> None: + _reject_tree_links(checkout, "OpenSAGE plugin") + _reject_python_bytecode(checkout, "OpenSAGE plugin") + observed = directory_tree_sha256(checkout, ignore_python_cache=True) + if observed != PLUGIN_TREE_SHA256: + raise RuntimeError( + "OpenSAGE plugin portable tree differs from its pinned source: " + f"observed {observed}" + ) + + def _ffmpeg_candidates(configured: Path | None, tools_root: Path) -> list[Path]: candidates: list[Path] = [] if configured: @@ -552,9 +688,29 @@ def _pin_ffmpeg(tools_root: Path, configured: Path | None) -> tuple[Path, Path]: source = path break if not source: - raise FileNotFoundError( - "pinned FFmpeg 8.1.1 was not found; pass bootstrap-tools --ffmpeg " - ) + with tempfile.TemporaryDirectory(prefix="ffmpeg-", dir=tools_root) as temporary: + staging = Path(temporary) + archive_path = staging / "ffmpeg.zip" + extracted = staging / "extracted" + _download_file( + FFMPEG_ARCHIVE_URL, + archive_path, + max_bytes=512 * 1024 * 1024, + ) + _require_hash(archive_path, FFMPEG_ARCHIVE_SHA256, "FFmpeg archive") + _extract_safe_zip(archive_path, extracted) + candidates = list(extracted.glob("*/bin/ffmpeg.exe")) + if len(candidates) != 1: + raise RuntimeError("FFmpeg archive has an unexpected layout") + source = candidates[0] + source_probe = source.with_name("ffprobe.exe") + _require_hash(source, FFMPEG_EXE_SHA256, "FFmpeg executable") + _require_hash(source_probe, FFPROBE_EXE_SHA256, "FFprobe executable") + destination_dir = tools_root / "ffmpeg-8.1.1" / "bin" + destination_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination_dir / "ffmpeg.exe") + shutil.copyfile(source_probe, destination_dir / "ffprobe.exe") + source = tools_root / "ffmpeg-8.1.1" / "bin" / "ffmpeg.exe" source_probe = source.with_name("ffprobe.exe") if not source_probe.is_file(): raise FileNotFoundError(f"ffprobe.exe is missing beside {source}") @@ -602,11 +758,7 @@ def bootstrap_tools(state_root: Path, ffmpeg_source: Path | None = None) -> dict f"found {sys.version.split()[0]}, {', '.join(dependency_versions)}" ) python_runtime = python_runtime_attestation() - if ( - python_runtime["launcher_sha256"] != PYTHON_LAUNCHER_SHA256 - or python_runtime["base_dll_sha256"] != PYTHON_BASE_DLL_SHA256 - or python_runtime["tree_sha256"] != PYTHON_RUNTIME_TREE_SHA256 - ): + if not python_runtime_identity_is_pinned(python_runtime): raise RuntimeError( "Python base runtime differs from the pinned 3.12.10 surface; " "reinstall the pinned interpreter before recreating the importer environment" @@ -659,6 +811,9 @@ def bootstrap_tools(state_root: Path, ffmpeg_source: Path | None = None) -> dict "opensage_w3d_plugin": { "source": PLUGIN_REPOSITORY, "commit": PLUGIN_COMMIT, + "archive": PLUGIN_ARCHIVE_URL, + "archive_sha256": PLUGIN_ARCHIVE_SHA256, + "tree_sha256": PLUGIN_TREE_SHA256, "python_bytecode_free": True, "license": "LGPL-3.0", "path": str(plugin), @@ -671,6 +826,8 @@ def bootstrap_tools(state_root: Path, ffmpeg_source: Path | None = None) -> dict }, "ffmpeg": { "version": FFMPEG_VERSION, + "archive": FFMPEG_ARCHIVE_URL, + "archive_sha256": FFMPEG_ARCHIVE_SHA256, "executable_sha256": FFMPEG_EXE_SHA256, "ffprobe_sha256": FFPROBE_EXE_SHA256, "license": "GPLv3 build", @@ -766,12 +923,7 @@ def tool_status( defusedxml_tree_ready = False try: python_runtime = python_runtime_attestation() - python_runtime_ready = ( - python_runtime["version"] == PYTHON_VERSION - and python_runtime["launcher_sha256"] == PYTHON_LAUNCHER_SHA256 - and python_runtime["base_dll_sha256"] == PYTHON_BASE_DLL_SHA256 - and python_runtime["tree_sha256"] == PYTHON_RUNTIME_TREE_SHA256 - ) + python_runtime_ready = python_runtime_identity_is_pinned(python_runtime) except (OSError, RuntimeError): python_runtime = {} python_runtime_ready = False @@ -785,6 +937,21 @@ def tool_status( ) except RuntimeError: plugin_clean = False + plugin_ready = False + if skip_w3d_attestation: + plugin_ready = (plugin / "io_mesh_w3d" / "__init__.py").is_file() + elif (plugin / ".git").exists(): + plugin_ready = ( + plugin_commit.casefold() == PLUGIN_COMMIT + and submodule_commit.casefold() == PLUGIN_SUBMODULE_COMMIT + and plugin_clean + ) + elif plugin.is_dir(): + try: + _attest_opensage_plugin_portable(plugin) + plugin_ready = True + except (OSError, RuntimeError): + plugin_ready = False blender_tree_ready = False if not skip_w3d_attestation and blender.is_file(): try: @@ -797,9 +964,7 @@ def tool_status( "blender_tree": blender_tree_ready, "opensage_w3d_plugin": ( (plugin / "io_mesh_w3d" / "__init__.py").is_file() - and plugin_commit.casefold() == PLUGIN_COMMIT - and submodule_commit.casefold() == PLUGIN_SUBMODULE_COMMIT - and plugin_clean + and plugin_ready ), "ffmpeg": ffmpeg.is_file() and sha256_file(ffmpeg).casefold() == FFMPEG_EXE_SHA256, "ffprobe": ffprobe.is_file() and sha256_file(ffprobe).casefold() == FFPROBE_EXE_SHA256, diff --git a/importer/openbfme_importer/catalog.py b/importer/openbfme_importer/catalog.py index 9904572..3864e56 100644 --- a/importer/openbfme_importer/catalog.py +++ b/importer/openbfme_importer/catalog.py @@ -40,6 +40,7 @@ ) KNOWN_SLICE_ARCHIVE_SHA256 = { + "_patch103.big": "4b9057b8c49053802797e22a171b378678f3b8f45edc9563c6480b254b968a1a", "ini.big": "e5e5aa2be5681161c2e24daa75e9294c38cb988133cba385bc433cba30fb72ca", "w3d.big": "c65fc670c35a2b938720a82328559786944b9dd3a37868f218f79226db3ed87d", "textures1.big": "defdcbef8bbd2b8b19571079c0224b2c89494410029860685ac221df7f57457a", @@ -253,19 +254,37 @@ def key(self) -> str: return self.name.casefold() -def archive_precedence(relative_path: str) -> tuple[int, str]: - """Lower numbers win when duplicate virtual paths exist.""" +_LAYER_DIRECTORY = re.compile(r"layer-(\d{1,4})(?:-[a-z0-9]+)?", re.IGNORECASE) + + +def archive_precedence(relative_path: str) -> tuple[int, int, str]: + """Lower tuples win when duplicate virtual paths exist. + + The leading component is the install layer: an expansion install mounts + after (and therefore shadows) its base game, so a layered install root + whose top-level directories are named ``layer-[-label]`` (junctions to + the real installs; the same naming the edition overlay uses) ranks every + layer-0 archive above every layer-1 archive regardless of archive family. + Plain single-install roots have no such directory and stay in layer 0 — + their ordering is unchanged. + """ + + layer = 0 + first = relative_path.split("/", 1)[0] + matched = _LAYER_DIRECTORY.fullmatch(first) + if matched is not None: + layer = int(matched.group(1)) name = Path(relative_path).name.casefold() patch_names = [value.casefold() for value in PATCH_ARCHIVES] if name in patch_names: - return patch_names.index(name), relative_path.casefold() + return layer, patch_names.index(name), relative_path.casefold() if "patch" in name and name.endswith(".big"): # Language patch archives use names such as EnglishPatch105.big. - return 50, relative_path.casefold() + return layer, 50, relative_path.casefold() # EA loads underscore override archives before normal data archives. if name.startswith("_"): - return 100, relative_path.casefold() - return 1000, relative_path.casefold() + return layer, 100, relative_path.casefold() + return layer, 1000, relative_path.casefold() def _directory_sha256( @@ -317,6 +336,25 @@ def _archive_directory_sha256( ) +def _payload_sample_sha256(path: Path) -> str: + """Cheap content canary: size + head/mid/tail samples (not full archive MD5).""" + + stat = path.stat() + size = int(stat.st_size) + digest = hashlib.sha256() + digest.update(str(size).encode("ascii")) + sample = 256 * 1024 + with path.open("rb") as stream: + digest.update(stream.read(sample)) + if size > sample * 2: + stream.seek(max(0, size // 2 - sample // 2)) + digest.update(stream.read(sample)) + if size > sample: + stream.seek(max(0, size - sample)) + digest.update(stream.read(sample)) + return digest.hexdigest() + + class InstallCatalog: FORMAT = 4 @@ -326,11 +364,17 @@ def __init__( archives: tuple[ArchiveInfo, ...], entries: tuple[CatalogEntry, ...], source_policy: ArchivePolicy | None = None, + payload_samples: Mapping[str, str] | None = None, ) -> None: self.install_root = install_root.resolve() self.archives = archives self.entries = entries self.source_policy = source_policy + self.payload_samples = { + key.casefold(): value.casefold() + for key, value in (payload_samples or {}).items() + if isinstance(key, str) and isinstance(value, str) + } by_key: dict[str, list[CatalogEntry]] = {} for entry in entries: by_key.setdefault(entry.key, []).append(entry) @@ -414,7 +458,19 @@ def build( CatalogEntry(relative, item.name, item.offset, item.size, precedence) for item in parsed.entries ) - return cls(root, tuple(archives), tuple(entries), source_policy) + samples = { + archive.relative_path: _payload_sample_sha256( + root / Path(archive.relative_path) + ) + for archive in archives + } + return cls( + root, + tuple(archives), + tuple(entries), + source_policy, + payload_samples=samples, + ) @classmethod def load(cls, path: Path | str) -> "InstallCatalog": @@ -461,7 +517,23 @@ def load(cls, path: Path | str) -> "InstallCatalog": if not isinstance(raw_policy, dict): raise ValueError("catalog source policy is invalid") source_policy = ArchivePolicy.from_serialized(raw_policy) - catalog = cls(Path(value["install_root"]), archives, entries, source_policy) + raw_samples = value.get("payload_samples") + samples: dict[str, str] = {} + if isinstance(raw_samples, dict): + for key, sample in raw_samples.items(): + if ( + isinstance(key, str) + and isinstance(sample, str) + and re.fullmatch(r"[0-9a-f]{64}", sample.casefold()) + ): + samples[key] = sample.casefold() + catalog = cls( + Path(value["install_root"]), + archives, + entries, + source_policy, + payload_samples=samples, + ) if source_policy is not None: expected = {item.path.casefold() for item in source_policy.archives} actual = {item.relative_path.casefold() for item in archives} @@ -470,12 +542,24 @@ def load(cls, path: Path | str) -> "InstallCatalog": return catalog def save(self, path: Path | str) -> None: + # Refresh samples at save so re-index always pins current canaries. + samples = { + archive.relative_path: _payload_sample_sha256( + self.install_root / Path(archive.relative_path) + ) + for archive in self.archives + if (self.install_root / Path(archive.relative_path)).is_file() + } + self.payload_samples = { + key.casefold(): value.casefold() for key, value in samples.items() + } value = { "format": self.FORMAT, "created_utc": datetime.now(timezone.utc).isoformat(), "install_root": str(self.install_root), "archives": [asdict(item) for item in self.archives], "entries": [asdict(item) for item in self.entries], + "payload_samples": samples, } if self.source_policy is not None: value["source_policy"] = self.source_policy.serialized() @@ -501,6 +585,11 @@ def stale_reasons(self) -> list[str]: current = indexed for relative in sorted(indexed - current): reasons.append(f"missing archive: {relative}") + deep = os.environ.get("OPENBFME_CATALOG_DEEP", "").strip().casefold() in { + "1", + "true", + "yes", + } for archive in self.archives: path = self.install_root / Path(archive.relative_path) if not path.is_file(): @@ -510,29 +599,8 @@ def stale_reasons(self) -> list[str]: if stat.st_size != archive.size or stat.st_mtime_ns != archive.mtime_ns: reasons.append(f"changed archive: {archive.relative_path}") continue - try: - current_directory = _archive_directory_sha256( - BigArchive.open(path), - archive.relative_path, - canonical_precedence[archive.relative_path.casefold()], - ) - except (OSError, ValueError) as exc: - reasons.append(f"unreadable archive: {archive.relative_path}: {exc}") - continue - if current_directory != archive.directory_sha256: - reasons.append(f"changed archive directory: {archive.relative_path}") - continue - if self.source_policy is not None: - members = { - item.path.casefold(): item for item in self.source_policy.archives - } - member = members.get(archive.relative_path.casefold()) - if member is None: - reasons.append(f"archive is outside source policy: {archive.relative_path}") - continue - if _md5_file(path) != member.md5: - reasons.append(f"changed archive payload: {archive.relative_path}") - continue + # Fast path: size+mtime already matched. Re-parse/MD5 the full 4GB + # policy set only when OPENBFME_CATALOG_DEEP=1 (or --deep tooling). catalog_entries = tuple( entry for entry in self.entries @@ -550,6 +618,55 @@ def stale_reasons(self) -> list[str]: != archive.directory_sha256 ): reasons.append(f"catalog directory mismatch: {archive.relative_path}") + continue + if self.source_policy is not None: + members = { + item.path.casefold(): item for item in self.source_policy.archives + } + member = members.get(archive.relative_path.casefold()) + if member is None: + reasons.append( + f"archive is outside source policy: {archive.relative_path}" + ) + continue + if deep and _md5_file(path) != member.md5: + reasons.append( + f"changed archive payload: {archive.relative_path}" + ) + continue + # Fast content canary: head/mid/tail sample. Catches many bit-flips + # without reading the full multi-GB archive set every CLI invoke. + expected_sample = self.payload_samples.get(archive.relative_path.casefold()) + if expected_sample: + try: + actual_sample = _payload_sample_sha256(path) + except OSError as exc: + reasons.append( + f"unreadable archive sample: {archive.relative_path}: {exc}" + ) + continue + if actual_sample != expected_sample: + reasons.append( + f"changed archive payload sample: {archive.relative_path}" + ) + continue + if deep: + try: + current_directory = _archive_directory_sha256( + BigArchive.open(path), + archive.relative_path, + canonical_precedence[archive.relative_path.casefold()], + ) + except (OSError, ValueError) as exc: + reasons.append( + f"unreadable archive: {archive.relative_path}: {exc}" + ) + continue + if current_directory != archive.directory_sha256: + reasons.append( + f"changed archive directory: {archive.relative_path}" + ) + continue return reasons @property @@ -612,7 +729,17 @@ def search(self, pattern: str) -> list[CatalogEntry]: ) def open_archive_for(self, entry: CatalogEntry) -> BigArchive: - archive = BigArchive.open(self.install_root / Path(entry.archive)) + # Reuse one BigArchive directory parse per archive path for the life of + # this catalog instance. Faction census used to re-open ini.big per doc. + cache = getattr(self, "_archive_handle_cache", None) + if cache is None: + self._archive_handle_cache: dict[str, BigArchive] = {} + cache = self._archive_handle_cache + key = entry.archive.casefold() + archive = cache.get(key) + if archive is None: + archive = BigArchive.open(self.install_root / Path(entry.archive)) + cache[key] = archive expected = (entry.name, entry.offset, entry.size) actual = {(item.name, item.offset, item.size) for item in archive.entries} if expected not in actual: diff --git a/importer/openbfme_importer/dependency_check.py b/importer/openbfme_importer/dependency_check.py new file mode 100644 index 0000000..2721379 --- /dev/null +++ b/importer/openbfme_importer/dependency_check.py @@ -0,0 +1,500 @@ +"""Unified dependency preflight for importer CLI + GUI. + +Checks BFME2 install shape/patch, pinned Python/Pillow tools, Blender/FFmpeg/ +OpenSAGE plugin, and optional Godot for pack launch/publish. +""" + +from __future__ import annotations + +import os +from pathlib import Path +import shutil +import sys +from typing import Any, Mapping, Sequence + +from .bootstrap import ( + BLENDER_EXE_SHA256, + BLENDER_VERSION, + DEFUSEDXML_VERSION, + FFMPEG_EXE_SHA256, + FFMPEG_VERSION, + FONTTOOLS_VERSION, + PLUGIN_COMMIT, + PYTHON_VERSION, + tool_status, +) +from .catalog import doctor_install +from .big import sha256_file +from .game import retail_game +from .paths import default_godot_content_root, default_state_root + + +CheckItem = dict[str, Any] + +# Modes that require W3D cook tools (Blender + plugin + ffmpeg). +_W3D_MODES = frozenset({"men-build", "build", "import-unit"}) +# Modes that only need install + Python/Pillow for plan/convert descriptors. +_CONVERT_MODES = frozenset( + {"faction-plan", "faction-convert", "import-faction", "plan", "convert"} +) + + +def _item( + *, + id: str, + label: str, + ok: bool, + required: bool, + detail: str, + fix: str = "", + expected: str = "", + found: str = "", +) -> CheckItem: + return { + "id": id, + "label": label, + "ok": bool(ok), + "required": bool(required), + "severity": "error" if required and not ok else ("ok" if ok else "warn"), + "detail": detail, + "fix": fix, + "expected": expected, + "found": found, + } + + +def _discover_godot() -> tuple[Path | None, str]: + """Locate Godot executable; returns (path|None, how_found).""" + + env = os.environ.get("OPENBFME_GODOT", "").strip() + if env: + path = Path(env).expanduser() + if path.is_file(): + return path.resolve(), "OPENBFME_GODOT" + # Portable user-local fallback used by the GUI / slice runner. + candidates = [ + Path.home() / "Downloads" / "godot47" / "Godot_v4.7-stable_win64.exe", + ] + which = shutil.which("godot") + if which: + candidates.insert(0, Path(which)) + for path in candidates: + if path.is_file(): + return path.resolve(), "discovered" + return None, "missing" + + +def _fast_tool_checks(state_root: Path) -> list[CheckItem]: + """Presence + pin-hash for executables; version strings for Python deps. + + Avoids full Blender tree re-hash (seconds) so the GUI stays responsive. + """ + + tools_root = Path(state_root).expanduser().resolve() / "tools" + items: list[CheckItem] = [] + + py_ver = sys.version.split()[0] + py_ok = py_ver == PYTHON_VERSION + items.append( + _item( + id="python", + label=f"Python {PYTHON_VERSION}", + ok=py_ok, + required=True, + expected=PYTHON_VERSION, + found=py_ver, + detail=f"interpreter {sys.executable}", + fix="run tools/bootstrap-importer-python.ps1 into the private state tools env", + ) + ) + + def _pkg(name: str, expected: str, import_name: str | None = None) -> CheckItem: + mod_name = import_name or name + try: + mod = __import__(mod_name) + found = str(getattr(mod, "__version__", "?")) + ok = found == expected + except ImportError: + found = "missing" + ok = False + return _item( + id=name, + label=f"{name} {expected}", + ok=ok, + required=True, + expected=expected, + found=found, + detail="pip package in importer env", + fix="recreate tools/python-3.12-env via bootstrap-importer-python.ps1", + ) + + items.append(_pkg("Pillow", "12.2.0", "PIL")) + items.append(_pkg("fontTools", FONTTOOLS_VERSION)) + items.append(_pkg("defusedxml", DEFUSEDXML_VERSION)) + + blender = tools_root / "blender-4.2.0-windows-x64" / "blender.exe" + if blender.is_file(): + try: + digest = sha256_file(blender).casefold() + blender_ok = digest == BLENDER_EXE_SHA256.casefold() + found = f"present sha={digest[:12]}…" + except OSError as exc: + blender_ok = False + found = f"unreadable: {exc}" + else: + blender_ok = False + found = "missing" + items.append( + _item( + id="blender", + label=f"Blender {BLENDER_VERSION} (pinned)", + ok=blender_ok, + required=True, + expected=BLENDER_EXE_SHA256[:16] + "…", + found=found, + detail=str(blender), + fix="run: openbfme-import bootstrap-tools", + ) + ) + + plugin = tools_root / "OpenSAGE.BlenderPlugin" / "io_mesh_w3d" / "__init__.py" + plugin_ok = plugin.is_file() + items.append( + _item( + id="w3d_plugin", + label="OpenSAGE W3D plugin", + ok=plugin_ok, + required=True, + expected=PLUGIN_COMMIT[:12], + found="present" if plugin_ok else "missing", + detail=str(plugin.parent.parent), + fix="run: openbfme-import bootstrap-tools", + ) + ) + + ffmpeg = tools_root / "ffmpeg-8.1.1" / "bin" / "ffmpeg.exe" + if ffmpeg.is_file(): + try: + digest = sha256_file(ffmpeg).casefold() + ffmpeg_ok = digest == FFMPEG_EXE_SHA256.casefold() + found = f"present sha={digest[:12]}…" + except OSError as exc: + ffmpeg_ok = False + found = f"unreadable: {exc}" + else: + ffmpeg_ok = False + found = "missing" + items.append( + _item( + id="ffmpeg", + label=f"FFmpeg {FFMPEG_VERSION} (pinned)", + ok=ffmpeg_ok, + required=True, + expected=FFMPEG_EXE_SHA256[:16] + "…", + found=found, + detail=str(ffmpeg), + fix="bootstrap-tools --ffmpeg ", + ) + ) + + git = shutil.which("git") + items.append( + _item( + id="git", + label="git (for plugin attest)", + ok=bool(git), + required=True, + found=git or "missing", + detail="needed to verify OpenSAGE plugin commit cleanliness", + fix="install Git for Windows and ensure it is on PATH", + ) + ) + return items + + +def _deep_tool_items(state_root: Path) -> list[CheckItem]: + status = tool_status(state_root, skip_w3d_attestation=False) + checks = status.get("checks", {}) + labels = { + "blender": f"Blender {BLENDER_VERSION} exe hash", + "blender_tree": "Blender portable tree hash", + "opensage_w3d_plugin": "OpenSAGE plugin commit + clean tree", + "ffmpeg": f"FFmpeg {FFMPEG_VERSION} hash", + "ffprobe": "FFprobe hash", + "python": f"Python {PYTHON_VERSION}", + "python_runtime": "Python runtime tree pin", + "pillow": "Pillow 12.2.0", + "pillow_tree": "Pillow package tree pin", + "fonttools": f"fontTools {FONTTOOLS_VERSION}", + "fonttools_tree": "fontTools tree pin", + "defusedxml": f"defusedxml {DEFUSEDXML_VERSION}", + "defusedxml_tree": "defusedxml tree pin", + } + items: list[CheckItem] = [] + for key, label in labels.items(): + ok = bool(checks.get(key)) + items.append( + _item( + id=f"deep:{key}", + label=label, + ok=ok, + required=True, + detail="deep tool_status attestation", + fix="bootstrap-tools / recreate python env" if not ok else "", + ) + ) + return items + + +def check_dependencies( + install: Path | str | None = None, + state_root: Path | str | None = None, + *, + mode: str = "faction-convert", + deep: bool = False, + godot_path: Path | str | None = None, + game: str = "bfme2", +) -> dict[str, Any]: + """Return a structured dependency report. + + *mode* selects which checks are **required** for Start: + - faction-plan / faction-convert: install + Python/Pillow (W3D tools warn) + - men-build / build: install + full cook toolchain; Godot warn for launch + """ + + install_path = ( + Path(install).expanduser() + if install is not None + else Path(os.environ.get("BFME2_INSTALL", r"F:\BFME2")) + ) + state = ( + Path(state_root).expanduser().resolve() + if state_root is not None + else default_state_root() + ) + mode_key = (mode or "faction-convert").strip().casefold() + game_definition = retail_game(game) + needs_w3d = mode_key in _W3D_MODES or mode_key == "men-build" + # Map GUI labels + if mode_key in {"men-build", "build"}: + needs_w3d = True + + items: list[CheckItem] = [] + + # --- Install --- + try: + doctor = doctor_install( + install_path, deep=False, game=game_definition.id + ) + except (OSError, ValueError, TypeError) as exc: + doctor = { + "ready": False, + "install_root": str(install_path), + "declared_patch": "unknown", + "missing_required": [str(exc)], + "executable_present": False, + } + install_ok = bool(doctor.get("ready")) + patch = str(doctor.get("declared_patch", "unknown")) + if game_definition.id == "bfme2": + patch_ok = patch.startswith("1.06") or patch == "1.06" + game_label = "BFME2" + expected_executable = "lotrbfme2.exe + core .big archives" + expected_patch = "1.06" + install_fix = "set Install path to a complete BFME2 1.06 folder" + patch_fix = "apply official 1.06 patch (or confirmed patch archives)" + patch_detail = "importer targets BFME2 1.06 retail" + else: + patch_ok = any( + patch.startswith(marker) or patch == marker + for marker in game_definition.patch_markers + ) + game_label = "RotWK" + expected_executable = "lotrbfme2ep1.exe + core .big archives" + expected_patch = "2.02 or 2.01" + install_fix = "set Install path to a complete RotWK 2.01/2.02 folder" + patch_fix = "apply a supported RotWK patch archive" + patch_detail = "importer analysis supports RotWK 2.01/2.02 retail" + # Prefer 1.06; warn (not hard fail) on older patches so doctor still runs. + items.append( + _item( + id="install", + label=f"{game_label} install present", + ok=install_ok, + required=True, + expected=expected_executable, + found=str(doctor.get("install_root", install_path)), + detail=( + f"patch={patch}; missing={doctor.get('missing_required') or []}" + ), + fix=install_fix, + ) + ) + items.append( + _item( + id="patch", + label=f"{game_label} patch {expected_patch}", + ok=patch_ok if install_ok else False, + required=True, + expected=expected_patch, + found=patch, + detail=patch_detail, + fix=patch_fix, + ) + ) + if doctor.get("executable_attestation", {}).get("modified_marker_detected"): + items.append( + _item( + id="modded_exe", + label="Unmodded executable preference", + ok=False, + required=False, + detail="DEV!ANCE / modified company marker on game.dat", + fix="prefer an unmodded install for deterministic catalogs", + ) + ) + + # --- Tools --- + if deep: + tool_items = _deep_tool_items(state) + else: + tool_items = _fast_tool_checks(state) + + for item in tool_items: + # Soften W3D tool requirements for plan/convert-only modes. + if not needs_w3d and item["id"] in { + "blender", + "w3d_plugin", + "ffmpeg", + "git", + "deep:blender", + "deep:blender_tree", + "deep:opensage_w3d_plugin", + "deep:ffmpeg", + "deep:ffprobe", + }: + item = dict(item) + item["required"] = False + if not item["ok"]: + item["severity"] = "warn" + items.append(item) + + # --- Godot (launch / publish target) --- + if godot_path is not None and str(godot_path).strip(): + gpath = Path(godot_path).expanduser() + g_ok = gpath.is_file() + g_found = str(gpath) if g_ok else "missing" + g_how = "configured" + else: + discovered, g_how = _discover_godot() + g_ok = discovered is not None + g_found = str(discovered) if discovered else "missing" + godot_required = needs_w3d # men-build wants Godot for launch CTA; not hard for cook + items.append( + _item( + id="godot", + label="Godot 4.x (slice launch)", + ok=g_ok, + required=False, # pack cook can finish without launching + expected="Godot 4.x stable win64", + found=f"{g_found} ({g_how})", + detail=f"content root default: {default_godot_content_root()}", + fix="set OPENBFME_GODOT to Godot_v4.x-stable_win64.exe", + ) + ) + + # --- State root writable --- + try: + state.mkdir(parents=True, exist_ok=True) + probe = state / ".openbfme-write-probe" + probe.write_text("ok", encoding="utf-8") + probe.unlink(missing_ok=True) + state_ok = True + state_detail = str(state) + except OSError as exc: + state_ok = False + state_detail = str(exc) + items.append( + _item( + id="state_root", + label="Private state root writable", + ok=state_ok, + required=True, + found=state_detail, + detail="caches, packs, reports", + fix="set OPENBFME_IMPORT_ROOT to a writable external/private path", + ) + ) + + errors = [i for i in items if i["required"] and not i["ok"]] + warnings = [i for i in items if (not i["required"]) and not i["ok"]] + return { + "schema": "openbfme.dependency-check", + "schemaVersion": 1, + "mode": mode_key, + "deep": bool(deep), + "install": doctor, + "state_root": str(state), + "ready": len(errors) == 0, + "error_count": len(errors), + "warning_count": len(warnings), + "items": items, + "errors": errors, + "warnings": warnings, + "summary": ( + f"{'READY' if not errors else 'BLOCKED'} · " + f"{sum(1 for i in items if i['ok'])}/{len(items)} checks ok · " + f"{len(errors)} errors · {len(warnings)} warnings" + ), + } + + +def format_dependency_report(report: Mapping[str, object]) -> str: + """Human-readable multi-line report for CLI/GUI log.""" + + lines = [str(report.get("summary", "dependency check")), ""] + items = report.get("items") + if not isinstance(items, list): + return "\n".join(lines) + for raw in items: + if not isinstance(raw, Mapping): + continue + mark = "OK " if raw.get("ok") else ("ERR" if raw.get("required") else "WRN") + label = raw.get("label", raw.get("id", "?")) + detail = raw.get("detail", "") + found = raw.get("found", "") + line = f"[{mark}] {label}" + if found: + line += f" — {found}" + if detail and str(detail) not in line: + line += f" ({detail})" + lines.append(line) + fix = raw.get("fix") + if fix and not raw.get("ok"): + lines.append(f" fix: {fix}") + return "\n".join(lines) + + +def blocking_message(report: Mapping[str, object]) -> str | None: + """Short dialog text if Start should be blocked; else None.""" + + if report.get("ready"): + return None + errors = report.get("errors") + if not isinstance(errors, list) or not errors: + return "Dependencies not ready." + lines = ["Cannot start — fix these first:\n"] + for raw in errors[:8]: + if not isinstance(raw, Mapping): + continue + lines.append(f"• {raw.get('label', raw.get('id'))}: {raw.get('found') or raw.get('detail')}") + if raw.get("fix"): + lines.append(f" → {raw['fix']}") + return "\n".join(lines) + + +__all__ = [ + "blocking_message", + "check_dependencies", + "format_dependency_report", +] diff --git a/importer/openbfme_importer/pipeline.py b/importer/openbfme_importer/pipeline.py index 7f1de34..c678e72 100644 --- a/importer/openbfme_importer/pipeline.py +++ b/importer/openbfme_importer/pipeline.py @@ -27,10 +27,13 @@ W3D_EXCLUDED_OPTIONAL_MESHES_OPTION, W3D_INPUT_RESOURCE_IDS_OPTION, W3D_PROVEN_NO_MOTION_ANIMATIONS_OPTION, + W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION, W3D_PROVEN_ROOT_RIGID_BAKE_OPTION, + W3D_RETAIL_ABSENT_TEXTURES_OPTION, W3D_TEXTURE_SUFFIXES, W3D_TEXTURE_OVERRIDES_OPTION, normalize_excluded_optional_meshes, + normalize_retail_absent_textures, normalize_texture_atlas_crops, normalize_w3d_no_motion_animations, normalize_w3d_texture_overrides, @@ -39,6 +42,7 @@ directory_tree_sha256, discover_executable, git_revision, + git_revision_at_exact_root, git_worktree_clean, inspect_tool, run_checked, @@ -53,7 +57,9 @@ RETAIL_PROVENANCE_CONTRACT = "openbfme.retail-import-provenance-v1" -MEN_FORDS_SOURCE_ENTRY_COUNT = 264 +RELEASE_IDENTITY_SCHEMA = "openbfme.bundled-source-identity" +# Exact resolved-source contract for the current 83-rule men-fords-v0 profile. +MEN_FORDS_SOURCE_ENTRY_COUNT = 335 MAX_RENDERED_OUTPUT_PATH = 512 W3D_ADAPTER_REPORT_CONTRACT = "openbfme.w3d-adapter-report" W3D_PRESENTATION_METADATA_CONTRACT = "openbfme.w3d-presentation-capabilities" @@ -343,6 +349,7 @@ def _validated_w3d_metadata( expected_excluded_optional_meshes: list[str] | None = None, expected_proven_root_rigid_bake: bool = False, expected_embedded_model_animation: bool = False, + expected_pivot_only_model: bool = False, ) -> dict[str, Any]: """Validate the private adapter report and return payload-free bundle facts.""" @@ -362,6 +369,8 @@ def _validated_w3d_metadata( raise ValueError("expected proven root-rigid bake must be a boolean") if not isinstance(expected_embedded_model_animation, bool): raise ValueError("expected embedded model animation must be a boolean") + if not isinstance(expected_pivot_only_model, bool): + raise ValueError("expected pivot-only model must be a boolean") if expected_proven_root_rigid_bake and asset_kind != "hierarchical": raise ValueError( "proven root-rigid bake is supported only for hierarchical W3D conversion" @@ -370,6 +379,14 @@ def _validated_w3d_metadata( raise ValueError( "embedded model animation is supported only for animated W3D conversion" ) + if expected_pivot_only_model and asset_kind != "hierarchical": + raise ValueError( + "proven pivot-only model is supported only for hierarchical W3D conversion" + ) + if expected_pivot_only_model and expected_proven_root_rigid_bake: + raise ValueError( + "proven pivot-only model cannot combine with proven root-rigid bake" + ) reported_asset_kind = report.get("asset_kind", "animated") if reported_asset_kind != asset_kind: raise RuntimeError( @@ -385,7 +402,7 @@ def _validated_w3d_metadata( "W3D adapter did not enforce the requested equipment semantics" ) - mesh_count = _report_int(report, "meshes", minimum=1) + mesh_count = _report_int(report, "meshes", minimum=0 if expected_pivot_only_model else 1) raw_root_rigid_bake = report.get("root_rigid_bake") root_rigid_keys = { "requested", @@ -443,6 +460,31 @@ def _validated_w3d_metadata( raise RuntimeError("W3D adapter reported an unexpected root-rigid bake") animated = asset_kind == "animated" skeletal = asset_kind in {"animated", "hierarchical"} and not root_rigid_applied + # The adapter proves which skeletal content the scene actually carried: + # skinned meshes must survive as exported skins, and a model rig must have + # skeleton-bound geometry. Proven rigid animated models (bone- or + # armature-parented meshes, no weights) legitimately export no skins; + # rigless animated composite carriers legitimately export neither. + model_skinned_mesh_count = report.get("skinned_meshes") + if ( + isinstance(model_skinned_mesh_count, bool) + or not isinstance(model_skinned_mesh_count, int) + or model_skinned_mesh_count < 0 + ): + raise RuntimeError("W3D adapter report has invalid skinned_meshes") + model_skeleton_count = report.get("skeletons") + if model_skeleton_count is None: + model_skeleton_count = 1 if animated else 0 + if ( + isinstance(model_skeleton_count, bool) + or not isinstance(model_skeleton_count, int) + or model_skeleton_count < 0 + ): + raise RuntimeError("W3D adapter report has an invalid skeleton count") + exported_skins_required = animated and model_skinned_mesh_count > 0 + exported_skeletal_meshes_required = animated and ( + model_skinned_mesh_count > 0 or model_skeleton_count > 0 + ) animation_count = _report_int(report, "animations", minimum=1 if animated else 0) animation_curve_count = _report_int( report, "animation_curves", minimum=1 if animated else 0 @@ -630,8 +672,11 @@ def _validated_w3d_metadata( expected_transform_animation_count == 0 and action_shape_exported_sampler_count != 0 ) - or (animated and action_shape_exported_skin_count < 1) - or (animated and action_shape_exported_skeletal_mesh_count < 1) + or (exported_skins_required and action_shape_exported_skin_count < 1) + or ( + exported_skeletal_meshes_required + and action_shape_exported_skeletal_mesh_count < 1 + ) or duplicated_logical_animation_count >= max(1, expected_transform_animation_count) or preserved_visibility_channel_count != expected_visibility_channels @@ -695,8 +740,11 @@ def _validated_w3d_metadata( or animation_count != 1 or embedded_counts["actionCount"] != action_shape_action_count or not embedded_transform_export_is_exact - or embedded_counts["exportedSkinCount"] < 1 - or embedded_counts["exportedSkeletalMeshCount"] < 1 + or (exported_skins_required and embedded_counts["exportedSkinCount"] < 1) + or ( + exported_skeletal_meshes_required + and embedded_counts["exportedSkeletalMeshCount"] < 1 + ) ): raise RuntimeError("W3D adapter embedded-animation proof is incomplete") elif any(embedded_counts.values()): @@ -754,13 +802,20 @@ def _validated_w3d_metadata( or split_animation_count > animation_count or split_counts["actionCount"] != split_animation_count * 2 or not split_transform_export_is_exact - or split_counts["exportedSkinCount"] < 1 - or split_counts["exportedSkeletalMeshCount"] < 1 + or (exported_skins_required and split_counts["exportedSkinCount"] < 1) + or ( + exported_skeletal_meshes_required + and split_counts["exportedSkeletalMeshCount"] < 1 + ) ): raise RuntimeError("W3D adapter split-animation proof is incomplete") elif any(split_counts.values()): raise RuntimeError("W3D adapter reported inconsistent split-animation proof") - bone_count = _report_int(report, "bones", minimum=1 if skeletal else 0) + bone_count = _report_int( + report, + "bones", + minimum=1 if skeletal and model_skinned_mesh_count > 0 else 0, + ) raw_skeleton_count = report.get("skeletons") if raw_skeleton_count is None: if asset_kind == "hierarchical": @@ -768,11 +823,23 @@ def _validated_w3d_metadata( skeleton_count = 1 if animated else 0 else: skeleton_count = _report_int(report, "skeletons") - expected_skeleton_count = 1 if skeletal else 0 - if skeleton_count != expected_skeleton_count: + if asset_kind == "animated": + # Rigless animated composite carriers are proven by the adapter when + # every clip keys its own auxiliary rig; any other animated model + # still carries exactly one model rig. + allowed_skeleton_counts = {0, 1} + elif skeletal: + allowed_skeleton_counts = {1} + else: + allowed_skeleton_counts = {0} + if skeleton_count not in allowed_skeleton_counts: raise RuntimeError("W3D adapter skeleton count does not match the asset kind") - vertex_count = _report_int(report, "vertices", minimum=1) - triangle_count = _report_int(report, "triangles", minimum=1) + vertex_count = _report_int( + report, "vertices", minimum=0 if expected_pivot_only_model else 1 + ) + triangle_count = _report_int( + report, "triangles", minimum=0 if expected_pivot_only_model else 1 + ) skinned_mesh_count = _report_int(report, "skinned_meshes") if not typed_action_report and animated and skinned_mesh_count < 1: raise RuntimeError("W3D adapter report has invalid skinned_meshes") @@ -1144,27 +1211,56 @@ def _w3d_conversion_cache_key( adapter_sha256: str, plugin_attestation_sha256: str, blender_tree_sha256: str, - argument_vector: list[str], + argument_vector: list[str] | None = None, + logical: Mapping[str, Any] | None = None, ) -> str: - """Hash every byte-affecting W3D conversion input canonically.""" + """Hash every byte-affecting W3D conversion input canonically. - payload = { - "adapter_sha256": adapter_sha256, - "argument_vector": list(argument_vector), - "blender_tree_sha256": blender_tree_sha256, - "plugin_attestation_sha256": plugin_attestation_sha256, - "source_hashes": { - name: source_hashes[name] - for name in sorted(source_hashes, key=lambda value: (value.casefold(), value)) - }, - } - return hashlib.sha256(_canonical_json_bytes(payload)).hexdigest() + Prefer *logical* identity (asset kind, model name, options) so absolute + output paths do not partition the shared DDC across factions/profiles. + """ + + if logical is not None: + identity: dict[str, Any] = { + "adapter_sha256": adapter_sha256, + "blender_tree_sha256": blender_tree_sha256, + "logical": dict(logical), + "plugin_attestation_sha256": plugin_attestation_sha256, + "source_hashes": { + name: source_hashes[name] + for name in sorted( + source_hashes, key=lambda value: (value.casefold(), value) + ) + }, + } + else: + identity = { + "adapter_sha256": adapter_sha256, + "argument_vector": list(argument_vector or ()), + "blender_tree_sha256": blender_tree_sha256, + "plugin_attestation_sha256": plugin_attestation_sha256, + "source_hashes": { + name: source_hashes[name] + for name in sorted( + source_hashes, key=lambda value: (value.casefold(), value) + ) + }, + } + return hashlib.sha256(_canonical_json_bytes(identity)).hexdigest() def _w3d_plugin_attestation_sha256(attestation: Mapping[str, str]) -> str: return hashlib.sha256(_canonical_json_bytes(dict(attestation))).hexdigest() +# The multi-job adapter captures each job's real process output into per-job +# files and rides it on the success marker as ``output_log`` (bounded at the +# adapter by MAX_JOB_OUTPUT_CAPTURE_BYTES). The warning-text guards in +# _finalize_w3d_bundle_job must evaluate that real content; an unbounded or +# missing log fails the job closed so it can never reach the conversion cache. +_W3D_MULTI_JOB_MAX_OUTPUT_LOG_CHARS = 1024 * 1024 + + def _entry_cache_key(entry: CatalogEntry) -> str: value = f"{entry.archive.casefold()}\n{entry.name.casefold()}\n{entry.offset}\n{entry.size}" return hashlib.sha256(value.encode("utf-8")).hexdigest()[:20] @@ -1175,6 +1271,31 @@ def _source_cache_key(entry: CatalogEntry, source_sha256: str) -> str: return hashlib.sha256(value.encode("utf-8")).hexdigest()[:20] +def _media_conversion_cache_key( + *, + source_sha256: str, + converter: str, + options: Mapping[str, Any], + tool_token: str, + relative_output: str = "", +) -> str: + """Content-addressed key for audio/texture outputs (shared across factions). + + Intentionally ignores pack-relative output path so the same source cooks + once for every faction/profile that needs it. + """ + + suffix = Path(relative_output).suffix.casefold() if relative_output else "" + payload = { + "converter": converter, + "options": dict(options), + "output_suffix": suffix, + "source_sha256": source_sha256.casefold(), + "tool_token": tool_token, + } + return hashlib.sha256(_canonical_json_bytes(payload)).hexdigest() + + def _w3d_staging_sources( resource: ResolvedResource, resources: tuple[ResolvedResource, ...], @@ -1268,7 +1389,7 @@ def _prepare_w3d_secondary_skin_streams( continue candidates.append((basename, result)) - if len(candidates) != 1: + if len(candidates) > 1: detail = "; ".join(rejected[:8]) if len(rejected) > 8: detail += f"; plus {len(rejected) - 8} more rejected candidates" @@ -1277,6 +1398,48 @@ def _prepare_w3d_secondary_skin_streams( f"found {len(candidates)}" + (f" ({detail})" if detail else "") ) + if not candidates: + # A bind-space coincidence rejection means redundancy could not be + # proven for the staged hierarchy. The pinned importer provably skips + # secondary chunks at read time (it seeks past VERTICES_2/NORMALS_2 + # unconditionally), so the conversion output is identical to the + # stripped outcome. Retain the streams and record every rejected + # candidate as exact evidence instead of inventing an equivalence + # proof that does not exist. Structural and identity failures (wrong + # hierarchy, malformed chunks, ambiguous providers) still fail + # closed, and when nothing was evaluated at all the job keeps failing. + unproven = [ + reason + for reason in rejected + if "bind position delta" in reason or "bind normal delta" in reason + ] + if len(unproven) != len(rejected) or not rejected: + detail = "; ".join(rejected[:8]) + if len(rejected) > 8: + detail += f"; plus {len(rejected) - 8} more rejected candidates" + raise RuntimeError( + "W3D secondary-skin proof requires exactly one compatible hierarchy; " + f"found 0 ({detail})" + ) + after_hashes = { + basename: sha256_file(path) for basename, path in sorted(copied.items()) + } + if after_hashes != before_hashes: + raise RuntimeError( + "W3D secondary-skin retention changed staged files" + ) + return { + "schema": "openbfme.w3d-secondary-skin-retention", + "schemaVersion": 0, + "retained": True, + "transformedMeshCount": 0, + "removedByteCount": 0, + "rejectedCandidates": rejected[:8], + "rejectedCandidateCount": len(rejected), + "stagedClosureBeforeSha256": _canonical_value_sha256(before_hashes), + "stagedClosureAfterSha256": _canonical_value_sha256(after_hashes), + } + hierarchy_basename, result = candidates[0] transformed = result.model_bytes() temporary = model.with_name(f".{model.name}.secondary-skin.tmp") @@ -1778,14 +1941,31 @@ def _validate_w3d_texture_override_glb( return {**proof, "entries": validated_entries, "complete": True} +def _strip_windows_extended_prefix(value: Path) -> Path: + """Normalize the \\\\?\\ and \\\\?\\UNC\\ forms Path.resolve may return.""" + text = str(value) + if text.startswith("\\\\?\\UNC\\"): + return Path("\\\\" + text[len("\\\\?\\UNC\\") :]) + if text.startswith("\\\\?\\"): + return Path(text[len("\\\\?\\") :]) + return value + + def _safe_output(root: Path, relative: str) -> Path: parts = safe_relative_parts(relative) - target = (root / Path(*parts)).resolve() + resolved_root = root.resolve() + target = (resolved_root / Path(*parts)).resolve() try: - target.relative_to(root.resolve()) + # Windows resolves a not-yet-created target through a different + # syscall path than the existing root, which can race concurrent + # directory creation and come back with a \\?\ extended prefix. The + # containment check must compare the same spelling on both sides. + contained = _strip_windows_extended_prefix(target).relative_to( + _strip_windows_extended_prefix(resolved_root) + ) except ValueError as exc: raise ValueError(f"output path escaped pack root: {relative!r}") from exc - return target + return resolved_root / contained def _render_output_template(template: str, *, index: int, stem: str, name: str) -> str: @@ -2080,12 +2260,18 @@ def _canonical_pack_inventory(pack_root: Path) -> list[dict[str, Any]]: def _importer_recipe_report() -> dict[str, Any]: root = repo_root_from_module() + release_identity_path = root / "release-identity.json" candidates = [ *sorted((root / "importer" / "openbfme_importer").rglob("*.py")), *sorted((root / "importer" / "blender").rglob("*.py")), - root / "importer" / "requirements-win.txt", + root / "importer" / ( + "requirements-release-win.txt" + if (root / "importer" / "requirements-release-win.txt").is_file() + else "requirements-win.txt" + ), root / "tools" / "openbfme_import.py", root / "tools" / "bootstrap-importer-python.ps1", + release_identity_path, ] files: list[dict[str, Any]] = [] digest = hashlib.sha256() @@ -2103,11 +2289,29 @@ def _importer_recipe_report() -> dict[str, Any]: digest.update(b"\0") digest.update(file_sha256.encode("ascii")) digest.update(b"\n") + if release_identity_path.is_file(): + identity = read_json(release_identity_path) + if not isinstance(identity, dict): + raise RuntimeError("bundled importer release identity is invalid") + candidate_commit = identity.get("commit") + candidate_clean = identity.get("sourceClean") + if ( + identity.get("schema") != RELEASE_IDENTITY_SCHEMA + or identity.get("schemaVersion") != 1 + or not _is_git_commit(candidate_commit) + or candidate_clean is not True + ): + raise RuntimeError("bundled importer release identity is invalid") + commit = candidate_commit + clean = candidate_clean + else: + commit = git_revision_at_exact_root(root) + clean = commit is not None and git_worktree_clean(root) return { "tree_sha256": digest.hexdigest(), "files": files, - "git_commit": git_revision(root), - "git_worktree_clean": git_worktree_clean(root), + "git_commit": commit, + "git_worktree_clean": clean, } @@ -2129,8 +2333,16 @@ def __init__( self.packs_root = self.workspace_root / "packs" self.reports_root = self.workspace_root / "reports" self.jobs_root = self.workspace_root / "jobs" - self.converted_cache_root = self.workspace_root / "cache" / "converted" - default_jobs = max(1, min(8, (os.cpu_count() or 1) - 2)) + # Shared DDC root (cross-faction): OPENBFME_SHARED_CACHE or workspace cache. + shared = os.environ.get("OPENBFME_SHARED_CACHE", "").strip() + if shared: + shared_root = Path(shared).expanduser().resolve() + else: + shared_root = self.workspace_root / "cache" + self.converted_cache_root = shared_root / "converted" + # Blender import/export is mostly single-threaded per process; more + # workers amortize spawn cost better on 16–24 core boxes. + default_jobs = max(1, min(16, (os.cpu_count() or 1) - 2)) if conversion_jobs is not None and conversion_jobs < 1: raise ValueError("conversion_jobs must be at least 1") self.conversion_jobs = conversion_jobs or default_jobs @@ -2141,7 +2353,16 @@ def __init__( self._w3d_batch_tools: dict[str, Any] | None = None self._w3d_final_attestation: dict[str, Any] | None = None self._blender_tree_verified = False + self._blender_exe_fingerprint: tuple[int, int] | None = None + self._blender_soft_tree_fingerprint_value: str | None = None self._python_runtime_report: dict[str, Any] = {} + self._ffmpeg_attested_path: str | None = None + self.media_cache_root = shared_root / "converted-media" + self.dev_mode = os.environ.get("OPENBFME_DEV", "").strip().casefold() in { + "1", + "true", + "yes", + } @property def conversion_cache_stats(self) -> dict[str, Any]: @@ -2152,6 +2373,118 @@ def conversion_cache_stats(self) -> dict[str, Any]: **self._conversion_cache_stats, } + def _ffmpeg_executable(self) -> Path | None: + """Resolve FFmpeg from this import state's pinned tools before PATH.""" + + configured = os.environ.get("OPENBFME_FFMPEG", "").strip() + if configured and Path(configured).is_file(): + return Path(configured).expanduser().resolve() + pinned = ( + self.state_root + / "tools" + / "ffmpeg-8.1.1" + / "bin" + / "ffmpeg.exe" + ) + if pinned.is_file(): + return pinned.resolve() + return discover_executable("ffmpeg", "OPENBFME_FFMPEG") + + def _media_cache_lock(self, key: str) -> threading.Lock: + with self._conversion_cache_lock: + return self._conversion_key_locks.setdefault(f"media:{key}", threading.Lock()) + + def _copy_media_cache_hit(self, key: str, target: Path) -> bool: + if not self.conversion_cache_enabled: + return False + entry = self.media_cache_root / key[:2] / key + metadata_path = entry / "metadata.json" + cached_output = entry / "output.bin" + try: + metadata = read_json(metadata_path) + if ( + metadata.get("format") != 1 + or metadata.get("key") != key + or metadata.get("output_size") != cached_output.stat().st_size + or metadata.get("output_sha256") != sha256_file(cached_output) + ): + if entry.is_dir(): + shutil.rmtree(entry) + with self._conversion_cache_lock: + self._conversion_cache_stats["misses"] += 1 + return False + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".media-cache-copying") + temporary.unlink(missing_ok=True) + shutil.copyfile(cached_output, temporary) + if ( + temporary.stat().st_size != metadata["output_size"] + or sha256_file(temporary) != metadata["output_sha256"] + ): + temporary.unlink(missing_ok=True) + raise RuntimeError("media conversion cache copy failed byte verification") + os.replace(temporary, target) + except (FileNotFoundError, KeyError, OSError, TypeError, ValueError): + if entry.is_dir(): + shutil.rmtree(entry, ignore_errors=True) + with self._conversion_cache_lock: + self._conversion_cache_stats["misses"] += 1 + return False + with self._conversion_cache_lock: + self._conversion_cache_stats["hits"] += 1 + return True + + def _populate_media_cache(self, key: str, target: Path) -> None: + if not self.conversion_cache_enabled: + return + destination = self.media_cache_root / key[:2] / key + if destination.is_dir(): + existing = destination / "output.bin" + if existing.is_file() and sha256_file(existing) == sha256_file(target): + return + raise RuntimeError( + "media conversion cache key produced non-byte-identical output" + ) + self.media_cache_root.mkdir(parents=True, exist_ok=True) + (self.media_cache_root / key[:2]).mkdir(parents=True, exist_ok=True) + temporary: Path | None = Path( + tempfile.mkdtemp(prefix=f".{key[:12]}.", dir=self.media_cache_root / key[:2]) + ) + try: + cached_output = temporary / "output.bin" + shutil.copyfile(target, cached_output) + output_sha256 = sha256_file(target) + if sha256_file(cached_output) != output_sha256: + raise RuntimeError("media conversion cache populate changed output bytes") + write_json_atomic( + temporary / "metadata.json", + { + "format": 1, + "key": key, + "output_size": target.stat().st_size, + "output_sha256": output_sha256, + }, + ) + try: + os.replace(temporary, destination) + except OSError: + if not destination.is_dir(): + raise + # A peer process populated the same key first: accept iff the + # peer's bytes are identical to ours (same key, same output). + existing = destination / "output.bin" + if not existing.is_file() or sha256_file(existing) != output_sha256: + raise RuntimeError( + "media conversion cache key produced non-byte-identical output" + ) + return + temporary = None + with self._conversion_cache_lock: + self._conversion_cache_stats["populated"] += 1 + finally: + if temporary is not None and temporary.is_dir(): + shutil.rmtree(temporary) + def _validate_source_catalog_binding(self, resolved: ResolvedProfile) -> str: profile = resolved.profile pack = profile.pack_metadata @@ -2648,23 +2981,43 @@ def build( force: bool = False, allow_incomplete: bool = False, ) -> Path: + from .progress import emit as progress_emit + self._validate_source_catalog_binding(resolved) if resolved.missing_required and not allow_incomplete: missing = ", ".join(resolved.missing_required) raise RuntimeError(f"required profile resources did not resolve: {missing}") + progress_emit("extract", "attesting archives/tools and extracting sources") source_archives = self._attest_source_archives(resolved) self._verify_required_tools(resolved) - extracted = self.extract_sources(resolved, force=force) pack_root = self.packs_root / resolved.profile.pack_id staging = self.packs_root / (resolved.profile.pack_id + ".building") if staging.exists(): shutil.rmtree(staging) staging.mkdir(parents=True) + progress_emit( + "convert-assets", + f"cooking pack resources ({len(resolved.resources)} rules)", + ) provenance_entries: list[dict[str, Any]] = [] incomplete: list[dict[str, str]] = [] - for resource in resolved.resources: + # Single classification pass: incomplete reasons + W3D jobs + media jobs. + w3d_jobs: list[ + tuple[int, list[Path], str | None, dict[str, Any], Path, str, str, str] + ] = [] + media_outputs: dict[tuple[int, int], list[Path]] = {} + media_errors: dict[tuple[int, int], Exception] = {} + media_jobs: list[ + tuple[int, int, Path, str, str | None, dict[str, Any], Path, str] + ] = [] + w3d_kind = { + "w3d-bundle": "animated", + "w3d-hierarchical": "hierarchical", + "w3d-static": "static", + } + for resource_index, resource in enumerate(resolved.resources): reasons: list[str] = [] if resource.missing_patterns: reasons.append( @@ -2679,15 +3032,8 @@ def build( {"resource": resource.rule.id, "reason": "; ".join(reasons)} ) - w3d_jobs: list[ - tuple[int, list[Path], str | None, dict[str, Any], Path, str, str, str] - ] = [] - for resource_index, resource in enumerate(resolved.resources): - if ( - resource.rule.converter - in {"w3d-bundle", "w3d-hierarchical", "w3d-static"} - and resource.entries - ): + converter = resource.rule.converter + if converter in w3d_kind and resource.entries: w3d_jobs.append( ( resource_index, @@ -2697,14 +3043,62 @@ def build( staging, resolved.profile.id, resource.rule.id, - { - "w3d-bundle": "animated", - "w3d-hierarchical": "hierarchical", - "w3d-static": "static", - }[resource.rule.converter], + w3d_kind[converter], + ) + ) + elif converter in {"audio", "texture", "texture-crop"}: + for entry_index, entry in enumerate(resource.entries): + cached = extracted.get( + (entry.archive.casefold(), entry.name.casefold()) + ) + if cached is None: + media_errors[(resource_index, entry_index)] = RuntimeError( + f"media source was not extracted: {entry.name}" + ) + continue + media_jobs.append( + ( + resource_index, + entry_index, + Path(cached["source_path"]), + converter, + resource.rule.output, + resource.rule.options, + staging, + str(cached.get("source_sha256") or ""), + ) ) + + # Overlap W3D and media lanes when both have work (independent outputs). + # Use one progress stage so concurrent workers do not clobber stage ETA. + if w3d_jobs and media_jobs: + from .progress import emit as progress_emit + + progress_emit( + "convert-assets", + f"w3d={len(w3d_jobs)} media={len(media_jobs)} (parallel)", + total_units=len(w3d_jobs) + len(media_jobs), + ) + with ThreadPoolExecutor(max_workers=2) as coordinator: + w3d_future = coordinator.submit( + self._convert_w3d_resources, + w3d_jobs, + progress_stage="", + ) + media_future = coordinator.submit( + self._convert_media_jobs, + media_jobs, + media_errors, + progress_stage="", + ) + w3d_outputs, w3d_errors = w3d_future.result() + media_outputs, media_errors = media_future.result() + else: + w3d_outputs, w3d_errors = self._convert_w3d_resources(w3d_jobs) + if media_jobs: + media_outputs, media_errors = self._convert_media_jobs( + media_jobs, media_errors ) - w3d_outputs, w3d_errors = self._convert_w3d_resources(w3d_jobs) for resource_index, resource in enumerate(resolved.resources): bundle_outputs: list[Path] | None = None @@ -2727,6 +3121,17 @@ def build( if resource.rule.required and not allow_incomplete: raise exc bundle_outputs = [] + elif resource_index not in w3d_outputs: + # Mirror the media lane: a required W3D job must never + # vanish between scheduling and collection without a + # recorded error (prior menofdale misclassification). + reason = "W3D conversion job was not scheduled" + incomplete.append( + {"resource": resource.rule.id, "reason": reason} + ) + if resource.rule.required and not allow_incomplete: + raise RuntimeError(reason) + bundle_outputs = [] else: bundle_outputs = w3d_outputs[resource_index] elif ( @@ -2819,6 +3224,27 @@ def build( # but their cooked files are declared only once. Repeating # outputs per source obscures collisions and inflates audits. output_paths = (bundle_outputs or []) if index == 0 else [] + elif resource.rule.converter in {"audio", "texture", "texture-crop"}: + media_key = (resource_index, index) + if media_key in media_errors: + exc = media_errors[media_key] + incomplete.append( + {"resource": resource.rule.id, "reason": str(exc)} + ) + if resource.rule.required and not allow_incomplete: + raise exc + output_paths = [] + elif media_key not in media_outputs: + # Must not silently emit empty outputs for a missed job. + reason = "media conversion job was not scheduled" + incomplete.append( + {"resource": resource.rule.id, "reason": reason} + ) + if resource.rule.required and not allow_incomplete: + raise RuntimeError(reason) + output_paths = [] + else: + output_paths = media_outputs[media_key] else: try: output_paths = self._convert_resource( @@ -2828,6 +3254,7 @@ def build( resource.rule.options, staging, index=index, + source_sha256=str(cache.get("source_sha256") or "") or None, ) except (FileNotFoundError, RuntimeError, ValueError) as exc: incomplete.append( @@ -2902,7 +3329,9 @@ def build( } provenance["bundle_files"] = _canonical_pack_inventory(staging) write_json_atomic(staging / "provenance" / "manifest.json", provenance) - audit = audit_pack(staging) + # On-disk audit.json is always full (canonical). Dev light audit is only + # an optional outer CLI speed path and must not claim hash validity here. + audit = audit_pack(staging, light=False) write_json_atomic(staging / "provenance" / "audit.json", audit) if not audit["valid"]: raise RuntimeError("built pack failed its internal hash audit") @@ -3003,7 +3432,11 @@ def _verify_required_tools(self, resolved: ResolvedProfile) -> None: self._python_runtime_report = dict(status.get("python_runtime", {})) def publish_to_godot( - self, pack_root: Path | str, content_root: Path | str + self, + pack_root: Path | str, + content_root: Path | str, + *, + allow_incomplete: bool = False, ) -> dict[str, str]: source = Path(pack_root).expanduser().resolve() pack_data = read_json(source / "pack.json") @@ -3014,10 +3447,15 @@ def publish_to_godot( ): raise ValueError(f"built pack has an unsafe id: {pack_id!r}") if not bool(pack_data.get("profile_build_complete", False)): - raise RuntimeError( - "incomplete retail packs cannot be published or selected" - ) - source_audit = audit_pack(source) + if not allow_incomplete: + raise RuntimeError( + "incomplete retail packs cannot be published or selected" + ) + # Dev/allow-incomplete path: still select so the vertical slice can + # load converted playableUnit/Structure registries while residual + # W3D gaps are fixed. Canonical release builds must stay complete. + # Publication always full-hash audits regardless of OPENBFME_DEV / light. + source_audit = audit_pack(source, light=False) if not source_audit["valid"]: raise RuntimeError("source pack failed canonical audit before publication") root = ensure_external_to_repo(Path(content_root), repo_root_from_module()) @@ -3035,7 +3473,7 @@ def publish_to_godot( if destination.is_dir(): if ( bundle_digest(destination) != digest - or not audit_pack(destination)["valid"] + or not audit_pack(destination, light=False)["valid"] ): raise RuntimeError( f"pre-existing published bundle is corrupt or tampered: {destination}" @@ -3051,28 +3489,53 @@ def publish_to_godot( raise RuntimeError( "published staging copy failed its bundle hash check" ) - if not audit_pack(staging)["valid"]: + if not audit_pack(staging, light=False)["valid"]: shutil.rmtree(staging) raise RuntimeError("published staging copy failed its canonical audit") os.replace(staging, destination) - selection = { + selection: dict[str, Any] = { "schema": "openbfme.pack-selection", "schemaVersion": 0, "activePack": relative.as_posix(), } - write_json_atomic(root / "selection.json", selection) + # Preserve supplemental packs (map overlays, ranger contracts, etc.) so + # a faction republish does not silently drop the rest of the slice stack. + selection_path = root / "selection.json" + if selection_path.is_file(): + try: + prior = read_json(selection_path) + except (OSError, ValueError, TypeError, KeyError): + prior = {} + prior_supplements = prior.get("supplementalPacks") + if isinstance(prior_supplements, list): + kept: list[str] = [] + seen: set[str] = set() + for raw in prior_supplements: + entry = str(raw).strip().replace("\\", "/") + if not entry or entry in seen: + continue + # Never re-attach the pack we just published as a supplement. + if entry == relative.as_posix() or entry.startswith(f"{pack_id}/"): + continue + if not (root / entry).is_dir(): + continue + seen.add(entry) + kept.append(entry) + if kept: + selection["supplementalPacks"] = kept + write_json_atomic(selection_path, selection) return { "bundle_sha256": digest, "published_pack": str(destination), - "selection": str(root / "selection.json"), + "selection": str(selection_path), "active_pack": relative.as_posix(), } def _canonical_tool_report(self) -> dict[str, Any]: report: dict[str, Any] = {} - ffmpeg = discover_executable("ffmpeg", "OPENBFME_FFMPEG") + ffmpeg = self._ffmpeg_executable() if ffmpeg: - inspected = inspect_tool("ffmpeg", "OPENBFME_FFMPEG") + inspected = inspect_tool("ffmpeg", executable=ffmpeg) report["ffmpeg"] = { "version": inspected.version, "sha256": sha256_file(ffmpeg), @@ -3158,6 +3621,7 @@ def _convert_resource( pack_root: Path, *, index: int, + source_sha256: str | None = None, ) -> list[Path]: relative_output = output or f"source/{source.name}" relative_output = _render_output_template( @@ -3168,120 +3632,275 @@ def _convert_resource( ) target = _safe_output(pack_root, relative_output) target.parent.mkdir(parents=True, exist_ok=True) - if converter == "hash-only": - return [] - if converter == "sage-map": - return self._convert_sage_map(source, target, options) - if converter == "sage-particle-definition": - return self._convert_sage_particle_definition(source, target, options) - if converter in {"copy", "text", "map"}: - shutil.copyfile(source, target) - return [target] - if converter in {"texture", "texture-crop"}: - try: - from PIL import Image - import PIL - except ImportError as exc: - raise FileNotFoundError( - "Pillow is required for deterministic DDS/TGA conversion" - ) from exc - if PIL.__version__ != "12.2.0": - raise RuntimeError( - f"Pillow 12.2.0 is required for deterministic texture output; found {PIL.__version__}" + match converter: + case "hash-only": + return [] + case "sage-map": + return self._convert_sage_map(source, target, options) + case "sage-particle-definition": + return self._convert_sage_particle_definition(source, target, options) + case "sage-scripts": + return self._convert_sage_scripts(source, target, options) + case "copy" | "text" | "map": + shutil.copyfile(source, target) + return [target] + case "texture" | "texture-crop" | "audio": + return self._convert_cached_media( + source, + converter, + options, + target, + relative_output=relative_output, + source_sha256=source_sha256, ) - if target.suffix.casefold() != ".png": - raise ValueError( - "deterministic texture conversion currently emits PNG only" + case "w3d-model" | "w3d-animation": + executable = os.environ.get("OPENBFME_W3D_CONVERTER", "").strip() + if not executable or not Path(executable).is_file(): + raise FileNotFoundError( + "W3D converter unavailable; set OPENBFME_W3D_CONVERTER" + ) + mode = "model" if converter == "w3d-model" else "animation" + run_checked( + [ + executable, + "convert", + "--mode", + mode, + "--input", + str(source), + "--output", + str(target), + ] ) - with Image.open(source) as opened: - converted = opened.convert("RGBA") - if converter == "texture-crop": - crop = options.get("crop", []) - if not ( - isinstance(crop, list) - and len(crop) == 4 - and all(isinstance(value, int) and value >= 0 for value in crop) - and crop[2] > 0 - and crop[3] > 0 - ): - raise ValueError( - "texture-crop requires options.crop=[x,y,width,height]" - ) - converted = converted.crop( - (crop[0], crop[1], crop[0] + crop[2], crop[1] + crop[3]) + if not target.is_file(): + raise RuntimeError(f"W3D converter did not create {target}") + return [target] + case _: + raise ValueError(f"unsupported converter: {converter}") + + def _png_compress_level(self) -> int: + """PNG zlib level. Default 9 (shipping) or 6 in OPENBFME_DEV. + + Tool token includes the level, so media DDC never mixes level-6 and level-9. + """ + + default = "6" if getattr(self, "dev_mode", False) else "9" + raw = os.environ.get("OPENBFME_PNG_LEVEL", default).strip() + try: + level = int(raw) + except ValueError as exc: + raise ValueError( + f"OPENBFME_PNG_LEVEL must be an integer 0-9, got {raw!r}" + ) from exc + if level < 0 or level > 9: + raise ValueError(f"OPENBFME_PNG_LEVEL must be 0-9, got {level}") + return level + + def _convert_media_jobs( + self, + media_jobs: list[ + tuple[int, int, Path, str, str | None, dict[str, Any], Path, str] + ], + prior_errors: dict[tuple[int, int], Exception] | None = None, + *, + progress_stage: str | None = "media", + ) -> tuple[dict[tuple[int, int], list[Path]], dict[tuple[int, int], Exception]]: + """Convert audio/texture jobs in parallel; returns outputs + errors.""" + + from .progress import emit as progress_emit + + outputs: dict[tuple[int, int], list[Path]] = {} + errors: dict[tuple[int, int], Exception] = dict(prior_errors or {}) + if not media_jobs: + return outputs, errors + stage = "" if progress_stage is None else progress_stage + if stage: + progress_emit( + stage, + f"converting {len(media_jobs)} audio/texture files " + f"({self.conversion_jobs} workers)", + total_units=len(media_jobs), + ) + + def _run_media( + job: tuple[int, int, Path, str, str | None, dict[str, Any], Path, str], + ) -> list[Path]: + ( + _resource_index, + entry_index, + source_path, + converter, + output, + options, + pack_root, + source_sha, + ) = job + return self._convert_resource( + source_path, + converter, + output, + options, + pack_root, + index=entry_index, + source_sha256=source_sha or None, + ) + + workers = min(self.conversion_jobs, len(media_jobs)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = { + pool.submit(_run_media, job): (job[0], job[1]) for job in media_jobs + } + for future in as_completed(futures): + key = futures[future] + try: + outputs[key] = future.result() + progress_emit( + stage, + f"media done ({len(outputs) + len(errors)}/{len(media_jobs)})", + unit_delta=1, ) - converted.save(target, format="PNG", compress_level=9, optimize=False) - return [target] + except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: + errors[key] = exc + progress_emit( + stage, + f"media failed ({len(outputs) + len(errors)}/{len(media_jobs)})", + unit_delta=1, + ) + return outputs, errors + + def _convert_cached_media( + self, + source: Path, + converter: str, + options: dict[str, Any], + target: Path, + *, + relative_output: str, + source_sha256: str | None = None, + ) -> list[Path]: + """Convert audio/texture with content-addressed cache + tool attest once.""" + + png_level = self._png_compress_level() if converter == "audio": - ffmpeg = discover_executable("ffmpeg", "OPENBFME_FFMPEG") + from .bootstrap import FFMPEG_EXE_SHA256 + + ffmpeg = self._ffmpeg_executable() if not ffmpeg: raise FileNotFoundError( "ffmpeg is required; set OPENBFME_FFMPEG to its executable" ) - from .bootstrap import FFMPEG_EXE_SHA256 - - if sha256_file(ffmpeg).casefold() != FFMPEG_EXE_SHA256: + resolved_ffmpeg = str(Path(ffmpeg).resolve()) + with self._conversion_cache_lock: + if self._ffmpeg_attested_path != resolved_ffmpeg: + if sha256_file(ffmpeg).casefold() != FFMPEG_EXE_SHA256: + raise RuntimeError( + "FFmpeg executable does not match the pinned 8.1.1 hash" + ) + self._ffmpeg_attested_path = resolved_ffmpeg + tool_token = f"ffmpeg:{FFMPEG_EXE_SHA256}" + else: + try: + import PIL + except ImportError as exc: + raise FileNotFoundError( + "Pillow is required for deterministic DDS/TGA conversion" + ) from exc + if PIL.__version__ != "12.2.0": raise RuntimeError( - "FFmpeg executable does not match the pinned 8.1.1 hash" + "Pillow 12.2.0 is required for deterministic texture output; " + f"found {PIL.__version__}" ) - command = [ - str(ffmpeg), - "-nostdin", - "-hide_banner", - "-loglevel", - "error", - "-y", - "-i", - str(source), - ] - if source.suffix.casefold() == target.suffix.casefold() and not bool( - options.get("force_pcm", False) - ): - shutil.copyfile(source, target) + tool_token = f"pillow:{PIL.__version__}:png{png_level}" + + source_sha = source_sha256 or sha256_file(source) + cache_key = _media_conversion_cache_key( + source_sha256=source_sha, + converter=converter, + options=options, + tool_token=tool_token, + relative_output=relative_output, + ) + with self._media_cache_lock(cache_key): + if self._copy_media_cache_hit(cache_key, target): return [target] - if target.suffix.casefold() != ".wav": - raise ValueError( - "deterministic audio conversion only supports exact copies or PCM WAV output" - ) - command.extend( - [ - "-fflags", - "+bitexact", - "-flags:a", - "+bitexact", - "-map_metadata", - "-1", - "-vn", - "-c:a", - "pcm_s16le", - ] - ) - command.append(str(target)) - run_checked(command) - return [target] - if converter in {"w3d-model", "w3d-animation"}: - executable = os.environ.get("OPENBFME_W3D_CONVERTER", "").strip() - if not executable or not Path(executable).is_file(): - raise FileNotFoundError( - "W3D converter unavailable; set OPENBFME_W3D_CONVERTER" - ) - mode = "model" if converter == "w3d-model" else "animation" - run_checked( - [ - executable, - "convert", - "--mode", - mode, - "--input", - str(source), - "--output", - str(target), - ] - ) - if not target.is_file(): - raise RuntimeError(f"W3D converter did not create {target}") - return [target] - raise ValueError(f"unsupported converter: {converter}") + match converter: + case "texture" | "texture-crop": + from PIL import Image + + if target.suffix.casefold() != ".png": + raise ValueError( + "deterministic texture conversion currently emits PNG only" + ) + with Image.open(source) as opened: + converted = opened.convert("RGBA") + if converter == "texture-crop": + crop = options.get("crop", []) + if not ( + isinstance(crop, list) + and len(crop) == 4 + and all( + isinstance(value, int) and value >= 0 + for value in crop + ) + and crop[2] > 0 + and crop[3] > 0 + ): + raise ValueError( + "texture-crop requires options.crop=" + "[x,y,width,height]" + ) + converted = converted.crop( + ( + crop[0], + crop[1], + crop[0] + crop[2], + crop[1] + crop[3], + ) + ) + converted.save( + target, + format="PNG", + compress_level=png_level, + optimize=False, + ) + case "audio": + ffmpeg = self._ffmpeg_executable() + assert ffmpeg is not None + if source.suffix.casefold() == target.suffix.casefold() and not bool( + options.get("force_pcm", False) + ): + shutil.copyfile(source, target) + else: + if target.suffix.casefold() != ".wav": + raise ValueError( + "deterministic audio conversion only supports exact " + "copies or PCM WAV output" + ) + command = [ + str(ffmpeg), + "-nostdin", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(source), + "-fflags", + "+bitexact", + "-flags:a", + "+bitexact", + "-map_metadata", + "-1", + "-vn", + "-c:a", + "pcm_s16le", + str(target), + ] + run_checked(command) + case _: + raise ValueError(f"unsupported media converter: {converter}") + self._populate_media_cache(cache_key, target) + return [target] def _convert_sage_particle_definition( self, @@ -3308,6 +3927,23 @@ def _convert_sage_particle_definition( write_json_atomic(target, particle_definition_document(definition)) return [target] + def _convert_sage_scripts( + self, + source: Path, + target: Path, + options: dict[str, Any], + ) -> list[Path]: + if not target.name.casefold().endswith(".scripts.json"): + raise ValueError("sage-scripts output must be a .scripts.json file") + if options: + raise ValueError( + "sage-scripts accepts no options; got: " + + ", ".join(sorted(options)) + ) + from .sage_scripts import convert_map_scripts + + return convert_map_scripts(source, target) + def _convert_sage_map( self, source: Path, @@ -3649,6 +4285,125 @@ def _convert_texture_atlas_crops( outputs.append(target) return outputs + def _blender_soft_tree_fingerprint(self, blender: Path) -> str: + """Bounded soft identity for end-of-batch tool checks (dev / opt-in). + + Portable Blender 4.x puts conversion-relevant scripts under + ``/scripts/**`` (not always a top-level ``scripts/``). Soft + mode samples: + - blender.exe + top-level natives + - every ``**/scripts/**`` tree (content hash for small files) + - bounded native libs under versioned subdirs (``.dll``/``.pyd``) + """ + + root = blender.parent + digest = hashlib.sha256() + seen: set[str] = set() + + def _add(path: Path, *, content: bool) -> None: + try: + if not path.is_file() or _is_link_like(path): + return + rel = path.relative_to(root).as_posix().casefold() + except (OSError, ValueError): + return + if rel in seen: + return + seen.add(rel) + try: + st = path.stat() + payload = path.read_bytes() if content and st.st_size <= 262_144 else None + except OSError: + return + digest.update(rel.encode("utf-8")) + digest.update(b"\0") + if payload is not None: + digest.update(hashlib.sha256(payload).digest()) + else: + digest.update(str(int(st.st_mtime_ns)).encode("ascii")) + digest.update(b"\0") + digest.update(str(int(st.st_size)).encode("ascii")) + digest.update(b"\n") + + _add(blender, content=False) + try: + for path in sorted(root.iterdir(), key=lambda p: p.name.casefold()): + if path.is_file() and path.suffix.casefold() in { + ".dll", + ".pyd", + ".exe", + }: + _add(path, content=False) + except OSError: + pass + + # Versioned portable layout: 4.2/scripts (not only top-level scripts/). + # Bound discovery to root + one-level children to stay O(version dirs). + script_roots: list[Path] = [] + try: + if (root / "scripts").is_dir() and not _is_link_like(root / "scripts"): + script_roots.append(root / "scripts") + for child in sorted(root.iterdir(), key=lambda p: p.name.casefold()): + if not child.is_dir() or _is_link_like(child): + continue + candidate = child / "scripts" + if candidate.is_dir() and not _is_link_like(candidate): + script_roots.append(candidate) + except OSError: + pass + + script_count = 0 + for scripts in script_roots: + try: + for path in sorted(scripts.rglob("*"), key=lambda p: str(p).casefold()): + if not path.is_file(): + continue + # Prefer content for scripts/python sources. + content = path.suffix.casefold() in { + ".py", + ".pyw", + ".txt", + ".xml", + ".json", + ".osl", + } + _add(path, content=content) + script_count += 1 + if script_count >= 800: + break + except OSError: + continue + if script_count >= 800: + break + + # Bounded natives under top-level and versioned children (e.g. 4.2/python). + native_count = 0 + native_roots: list[Path] = [root] + try: + for child in sorted(root.iterdir(), key=lambda p: p.name.casefold()): + if child.is_dir() and not _is_link_like(child): + native_roots.append(child) + except OSError: + pass + for native_root in native_roots: + try: + for path in sorted( + native_root.rglob("*"), key=lambda p: str(p).casefold() + ): + if not path.is_file(): + continue + if path.suffix.casefold() not in {".dll", ".pyd"}: + continue + _add(path, content=False) + native_count += 1 + if native_count >= 200: + break + except OSError: + continue + if native_count >= 200: + break + return digest.hexdigest() + def _prepare_w3d_execution_tools( self, blender: Path, plugin: Path ) -> tuple[str, dict[str, str]]: @@ -3662,6 +4417,17 @@ def _prepare_w3d_execution_tools( blender_tree_sha256 = prepare_blender_portable_tree(self.state_root, blender) plugin_attestation = prepare_opensage_plugin_checkout(self.state_root, plugin) self._blender_tree_verified = True + try: + st = blender.stat() + # Keep legacy exe pair for diagnostics; soft end-attest uses the + # broader scripts/native sample fingerprint. + self._blender_exe_fingerprint = (int(st.st_mtime_ns), int(st.st_size)) + self._blender_soft_tree_fingerprint_value = ( + self._blender_soft_tree_fingerprint(blender) + ) + except OSError: + self._blender_exe_fingerprint = None + self._blender_soft_tree_fingerprint_value = None return blender_tree_sha256, plugin_attestation def _w3d_execution_tool_paths(self) -> tuple[Path, Path]: @@ -3723,7 +4489,7 @@ def _end_w3d_conversion_batch(self) -> None: self._w3d_batch_tools = None from .bootstrap import ( BLENDER_TREE_SHA256, - _attest_opensage_plugin_checkout, + prepare_opensage_plugin_checkout, _reject_python_bytecode, _reject_tree_links, ) @@ -3732,21 +4498,56 @@ def _end_w3d_conversion_batch(self) -> None: plugin = Path(tools["plugin"]) _reject_tree_links(blender.parent, "Blender portable tree") _reject_python_bytecode(blender.parent, "Blender portable tree") - plugin_attestation = _attest_opensage_plugin_checkout(plugin) - if directory_tree_sha256(blender.parent) != BLENDER_TREE_SHA256: + plugin_attestation = prepare_opensage_plugin_checkout(self.state_root, plugin) + # Shipping default: full tree re-hash at end (~5–7s). Soft end-attest + # (bounded scripts/native sample) is only for OPENBFME_DEV or explicit + # OPENBFME_SOFT_TOOL_ATTEST. OPENBFME_STRICT_TOOL_ATTEST forces full. + force_strict = os.environ.get( + "OPENBFME_STRICT_TOOL_ATTEST", "" + ).strip().casefold() in {"1", "true", "yes"} + allow_soft = ( + not force_strict + and ( + self.dev_mode + or os.environ.get("OPENBFME_SOFT_TOOL_ATTEST", "") + .strip() + .casefold() + in {"1", "true", "yes"} + ) + ) + tree_ok = False + soft_fp = getattr(self, "_blender_soft_tree_fingerprint_value", None) + if allow_soft and soft_fp is not None: + try: + tree_ok = soft_fp == self._blender_soft_tree_fingerprint( + blender + ) and str(tools.get("blender_tree_sha256", "")).casefold() == ( + BLENDER_TREE_SHA256.casefold() + ) + except OSError: + tree_ok = False + if not tree_ok and directory_tree_sha256(blender.parent) != BLENDER_TREE_SHA256: raise RuntimeError("Blender portable tree changed during W3D conversion") - plugin_clean = git_worktree_clean(plugin) + # Archive-provisioned plugins intentionally have no Git metadata. + # prepare_opensage_plugin_checkout() has already re-hashed that entire + # portable tree above, which is the equivalent cleanliness proof. + plugin_clean = ( + git_worktree_clean(plugin) if (plugin / ".git").exists() else True + ) if not plugin_clean: raise RuntimeError("OpenSAGE W3D plugin changed during W3D conversion") self._w3d_final_attestation = { "blender_tree_sha256": BLENDER_TREE_SHA256, "plugin": plugin_attestation, "plugin_worktree_clean": plugin_clean, + "end_attest": "soft" if (allow_soft and tree_ok) else "full", } def _convert_w3d_resources( self, jobs: list[tuple[int, list[Path], str | None, dict[str, Any], Path, str, str, str]], + *, + progress_stage: str | None = "blender-w3d", ) -> tuple[dict[int, list[Path]], dict[int, Exception]]: outputs: dict[int, list[Path]] = {} errors: dict[int, Exception] = {} @@ -3777,21 +4578,262 @@ def _convert_w3d_resources( validated_output.parent.mkdir(parents=True, exist_ok=True) self._begin_w3d_conversion_batch() try: - with ThreadPoolExecutor(max_workers=min(self.conversion_jobs, len(jobs))) as pool: - futures = { - pool.submit(self._convert_w3d_bundle, *job[1:]): job[0] - for job in jobs - } - for future in as_completed(futures): - index = futures[future] - try: - outputs[index] = future.result() - except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: - errors[index] = exc + from .progress import emit as progress_emit + + use_multi = os.environ.get("OPENBFME_W3D_MULTI", "1").strip().casefold() not in { + "0", + "false", + "no", + } + try: + batch_size = max(1, int(os.environ.get("OPENBFME_W3D_BATCH_SIZE", "8"))) + except ValueError: + batch_size = 8 + stage = "" if progress_stage is None else progress_stage + if stage: + progress_emit( + stage, + f"converting {len(jobs)} W3D models " + f"(workers={self.conversion_jobs}, multi={use_multi}, batch={batch_size})", + total_units=len(jobs), + ) + if use_multi and len(jobs) > 1 and batch_size > 1: + chunks = [ + jobs[offset : offset + batch_size] + for offset in range(0, len(jobs), batch_size) + ] + workers = min(self.conversion_jobs, len(chunks)) + with ThreadPoolExecutor(max_workers=workers) as pool: + futures = { + pool.submit(self._convert_w3d_chunk, chunk): chunk + for chunk in chunks + } + for future in as_completed(futures): + try: + chunk_outputs, chunk_errors = future.result() + except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: + # Whole chunk failed before per-job accounting. + for job in futures[future]: + errors[job[0]] = exc + progress_emit( + stage, + f"chunk failed ({len(outputs) + len(errors)}/{len(jobs)})", + unit_delta=1, + ) + continue + outputs.update(chunk_outputs) + errors.update(chunk_errors) + done = len(chunk_outputs) + len(chunk_errors) + progress_emit( + stage, + f"chunk done +{done} ({len(outputs) + len(errors)}/{len(jobs)})", + unit_delta=done, + ) + else: + with ThreadPoolExecutor( + max_workers=min(self.conversion_jobs, len(jobs)) + ) as pool: + futures = { + pool.submit(self._convert_w3d_bundle, *job[1:]): job[0] + for job in jobs + } + for future in as_completed(futures): + index = futures[future] + try: + outputs[index] = future.result() + progress_emit( + stage, + f"model done ({len(outputs) + len(errors)}/{len(jobs)})", + unit_delta=1, + ) + except ( + FileNotFoundError, + RuntimeError, + ValueError, + OSError, + ) as exc: + errors[index] = exc + progress_emit( + stage, + f"model failed ({len(outputs) + len(errors)}/{len(jobs)})", + unit_delta=1, + ) finally: self._end_w3d_conversion_batch() return outputs, errors + def _convert_w3d_chunk( + self, + chunk: list[ + tuple[int, list[Path], str | None, dict[str, Any], Path, str, str, str] + ], + ) -> tuple[dict[int, list[Path]], dict[int, Exception]]: + """Convert one batch of W3D jobs; one Blender process for cache misses.""" + + outputs: dict[int, list[Path]] = {} + errors: dict[int, Exception] = {} + # Fall back to single-job path when batch is tiny or multi is unsafe. + if len(chunk) == 1: + index = chunk[0][0] + try: + outputs[index] = self._convert_w3d_bundle(*chunk[0][1:]) + except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: + errors[index] = exc + return outputs, errors + + prepared: list[dict[str, Any]] = [] + for job in chunk: + index = job[0] + try: + prepared.append( + self._prepare_w3d_bundle_job(index, *job[1:]) + ) + except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: + errors[index] = exc + + hits = [item for item in prepared if item["cache_hit"]] + misses = [item for item in prepared if not item["cache_hit"]] + for item in hits: + try: + outputs[item["index"]] = self._finalize_w3d_bundle_job( + item, item["combined_log"], cache_hit=True + ) + except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: + errors[item["index"]] = exc + + if not misses: + return outputs, errors + + if self._w3d_batch_tools is None: + raise RuntimeError("W3D conversion requires an active attested batch") + blender = Path(self._w3d_batch_tools["blender"]) + plugin = Path(self._w3d_batch_tools["plugin"]) + multi_adapter = ( + repo_root_from_module() / "importer" / "blender" / "w3d_multi_to_glb.py" + ) + batch_root = ( + self.jobs_root + / misses[0]["profile_id"] + / "w3d-multi" + / hashlib.sha256( + "|".join(item["asset_id"] for item in misses).encode("utf-8") + ).hexdigest()[:16] + ) + if batch_root.exists(): + shutil.rmtree(batch_root) + batch_root.mkdir(parents=True) + multi_jobs = [] + for item in misses: + multi_jobs.append( + { + "job_id": item["asset_id"], + "model": str(item["model"]), + "asset_kind": item["asset_kind"], + "animations": [str(path) for path in item["animations"]], + "required_equipment": list(item["required_equipment"]), + "excluded_optional_meshes": list(item["excluded_optional_meshes"]), + "proven_root_rigid_bake": item["proven_root_rigid_bake"], + "proven_pivot_only_model": item.get( + "proven_pivot_only_model", False + ), + "retail_absent_textures": list( + item.get("retail_absent_textures", []) + ), + "output": str(item["target"]), + } + ) + jobs_path = batch_root / "jobs.json" + write_json_atomic( + jobs_path, + {"schema": "openbfme.w3d-multi-jobs", "jobs": multi_jobs}, + ) + command = [ + str(blender), + "--factory-startup", + "-noaudio", + "--background", + "--python-use-system-env", + "--python-exit-code", + "1", + "--python", + str(multi_adapter), + "--", + "--plugin-root", + str(plugin), + "--jobs", + str(jobs_path), + ] + isolated = _isolated_blender_environment(os.environ, batch_root) + try: + result = run_checked(command, env=isolated) + combined = result.stdout + "\n" + result.stderr + except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: + for item in misses: + errors[item["index"]] = exc + return outputs, errors + + ok_payloads: dict[str, dict[str, Any]] = {} + fails: dict[str, str] = {} + for line in combined.splitlines(): + if line.startswith("OPENBFME_W3D_JOB_OK "): + payload = json.loads(line.split(" ", 1)[1]) + ok_payloads[str(payload["job_id"])] = payload + elif line.startswith("OPENBFME_W3D_JOB_FAIL "): + payload = json.loads(line.split(" ", 1)[1]) + detail = str( + payload.get("error") or payload.get("error_type") or "failed" + ) + failure_phase = payload.get("failure_phase") + failure_kind = payload.get("failure_kind") + if ( + type(failure_phase) is str + and failure_phase + and type(failure_kind) is str + and failure_kind + ): + detail = f"{detail} [failure_phase={failure_phase} failure_kind={failure_kind}]" + fails[str(payload["job_id"])] = detail + + for item in misses: + asset_id = item["asset_id"] + if asset_id in fails: + errors[item["index"]] = RuntimeError( + f"W3D multi-job failed for {asset_id}: {fails[asset_id]}" + ) + continue + payload = ok_payloads.get(asset_id) + if payload is None: + errors[item["index"]] = RuntimeError( + f"W3D multi-job missing success marker for {asset_id}" + ) + continue + report = payload.get("report") + output_log = payload.get("output_log") + if ( + not isinstance(report, dict) + or type(output_log) is not str + or len(output_log) > _W3D_MULTI_JOB_MAX_OUTPUT_LOG_CHARS + ): + errors[item["index"]] = RuntimeError( + f"W3D multi-job emitted an invalid or unbounded output log " + f"for {asset_id}" + ) + continue + # Finalize against the job's REAL captured output (plus the + # synthesized success marker), never the marker alone: the + # warning-text guards in _finalize_w3d_bundle_job must see the + # same content the single-job process log carries, and the + # combined log is what the conversion cache stores for later + # single-job cache hits. + log = output_log + "\nOPENBFME_W3D_OK " + json.dumps(report, sort_keys=True) + try: + outputs[item["index"]] = self._finalize_w3d_bundle_job( + item, log, cache_hit=False + ) + except (FileNotFoundError, RuntimeError, ValueError, OSError) as exc: + errors[item["index"]] = exc + return outputs, errors + def _w3d_cache_lock(self, key: str) -> threading.Lock: with self._conversion_cache_lock: return self._conversion_key_locks.setdefault(key, threading.Lock()) @@ -3890,8 +4932,9 @@ def _populate_w3d_cache(self, key: str, target: Path, combined_log: str) -> None if temporary is not None and temporary.is_dir(): shutil.rmtree(temporary) - def _convert_w3d_bundle( + def _prepare_w3d_bundle_job( self, + index: int, staging_sources: list[Path], output: str | None, options: dict[str, Any], @@ -3899,7 +4942,9 @@ def _convert_w3d_bundle( profile_id: str, asset_id: str, asset_kind: str, - ) -> list[Path]: + ) -> dict[str, Any]: + """Stage sources and resolve cache for one W3D job (no Blender yet).""" + if not output: raise ValueError(f"w3d-{asset_kind} requires an output path") if asset_kind not in {"animated", "hierarchical", "static"}: @@ -3933,11 +4978,28 @@ def _convert_w3d_bundle( raise ValueError( "proven root-rigid bake is supported only for hierarchical W3D conversion" ) + proven_pivot_only_model = options.get( + W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION, False + ) + if not isinstance(proven_pivot_only_model, bool): + raise ValueError( + f"W3D options.{W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION} must be a boolean" + ) + if proven_pivot_only_model and asset_kind != "hierarchical": + raise ValueError( + "proven pivot-only model is supported only for hierarchical W3D conversion" + ) + if proven_pivot_only_model and proven_root_rigid_bake: + raise ValueError( + "proven pivot-only model cannot combine with proven root-rigid bake" + ) + retail_absent_textures = normalize_retail_absent_textures( + options.get(W3D_RETAIL_ABSENT_TEXTURES_OPTION, []) + ) if asset_kind != "animated" and required_equipment: raise ValueError( f"w3d-{asset_kind} does not accept options.required_equipment" ) - if self._w3d_batch_tools is None: raise RuntimeError("W3D conversion requires an active attested batch") blender = Path(self._w3d_batch_tools["blender"]) @@ -3948,7 +5010,6 @@ def _convert_w3d_bundle( shutil.rmtree(job_root) input_root = job_root / "input" copied = _stage_w3d_sources(staging_sources, input_root) - model = copied.get(model_name) if not model: raise FileNotFoundError( @@ -3959,20 +5020,45 @@ def _convert_w3d_bundle( model, options.get(W3D_PROVEN_NO_MOTION_ANIMATIONS_OPTION), ) - secondary_skin_proof = _prepare_w3d_secondary_skin_streams(copied, model) + try: + secondary_skin_proof = _prepare_w3d_secondary_skin_streams(copied, model) + except RuntimeError as exc: + raise RuntimeError( + f"W3D secondary-skin preparation failed for asset '{asset_id}' " + f"model '{model.name}': {exc}" + ) from exc texture_override_proof = _apply_w3d_texture_overrides( copied, model, options.get(W3D_TEXTURE_OVERRIDES_OPTION), ) animations: list[Path] = [] + effective_animation_names: list[str] = [] + empty_placeholder_animations: list[str] = [] for name in animation_names: animation = copied.get(name) if not animation: raise FileNotFoundError( f"W3D animation was not selected by the profile: {name}" ) + # Retail ships a handful of zero-byte W3D placeholders (e.g. + # rugimli_idlg.w3d, guboromir_dieb.w3d). Importing them creates no + # owned action and used to surface as a misleading owner-rig error. + # Drop them from the conversion set with explicit evidence; do not + # invent clips. + if animation.stat().st_size == 0: + empty_placeholder_animations.append(name) + continue animations.append(animation) + effective_animation_names.append(name) + if asset_kind == "animated" and not animations: + raise ValueError( + "w3d-bundle requires at least one non-empty animation; " + "all declared clips were zero-byte retail placeholders: " + + ", ".join(empty_placeholder_animations) + ) + # Cache / finalize must key on the clips actually converted. + animation_names = effective_animation_names target = _safe_output(pack_root, output) target.parent.mkdir(parents=True, exist_ok=True) @@ -3995,6 +5081,7 @@ def _convert_w3d_bundle( "--asset-kind", asset_kind, *(["--proven-root-rigid-bake"] if proven_root_rigid_bake else []), + *(["--proven-pivot-only-model"] if proven_pivot_only_model else []), "--output", str(target), "--animations", @@ -4003,8 +5090,9 @@ def _convert_w3d_bundle( *required_equipment, "--excluded-optional-meshes", *excluded_optional_meshes, + "--retail-absent-textures", + *retail_absent_textures, ] - isolated_environment = _isolated_blender_environment(os.environ, job_root) source_hashes = { name: sha256_file(path) for name, path in sorted( @@ -4013,21 +5101,75 @@ def _convert_w3d_bundle( } cache_key = _w3d_conversion_cache_key( source_hashes=source_hashes, - adapter_sha256=sha256_file(adapter), + adapter_sha256=( + sha256_file(adapter) + + sha256_file( + repo_root_from_module() + / "importer" + / "blender" + / "w3d_multi_to_glb.py" + ) + ), plugin_attestation_sha256=str( self._w3d_batch_tools["plugin_attestation_sha256"] ), blender_tree_sha256=str(self._w3d_batch_tools["blender_tree_sha256"]), - argument_vector=[*command, "--canonical-options", json.dumps(options, sort_keys=True, separators=(",", ":"))], + logical={ + "asset_kind": asset_kind, + "model_name": model_name, + "animation_names": list(animation_names), + "required_equipment": list(required_equipment), + "excluded_optional_meshes": list(excluded_optional_meshes), + "proven_root_rigid_bake": proven_root_rigid_bake, + "options": json.loads( + json.dumps(options, sort_keys=True, separators=(",", ":")) + ), + }, ) - cache_hit = False + combined_log: str | None with self._w3d_cache_lock(cache_key): combined_log = self._copy_w3d_cache_hit(cache_key, target) - if combined_log is None: - result = run_checked(command, env=isolated_environment) - combined_log = result.stdout + "\n" + result.stderr - else: - cache_hit = True + return { + "index": index, + "asset_id": asset_id, + "profile_id": profile_id, + "asset_kind": asset_kind, + "model_name": model_name, + "animation_names": animation_names, + "empty_placeholder_animations": empty_placeholder_animations, + "required_equipment": required_equipment, + "excluded_optional_meshes": excluded_optional_meshes, + "proven_root_rigid_bake": proven_root_rigid_bake, + "proven_pivot_only_model": proven_pivot_only_model, + "retail_absent_textures": retail_absent_textures, + "model": model, + "animations": animations, + "target": target, + "copied": copied, + "options": options, + "pack_root": pack_root, + "report_relative_path": report_relative_path, + "no_motion_proof": no_motion_proof, + "secondary_skin_proof": secondary_skin_proof, + "texture_override_proof": texture_override_proof, + "cache_key": cache_key, + "command": command, + "job_root": job_root, + "cache_hit": combined_log is not None, + "combined_log": combined_log or "", + } + + def _finalize_w3d_bundle_job( + self, + prepared: Mapping[str, Any], + combined_log: str, + *, + cache_hit: bool, + ) -> list[Path]: + """Validate GLB + adapter report and write metrics/cache.""" + + target = Path(prepared["target"]) + pack_root = Path(prepared["pack_root"]) unsupported = [ line for line in combined_log.splitlines() @@ -4062,21 +5204,22 @@ def _convert_w3d_bundle( report = json.loads(marker_lines[0].split(" ", 1)[1]) metrics = _validated_w3d_metadata( report, - required_equipment, - expected_animation_count=len(animation_names), - asset_kind=asset_kind, - expected_excluded_optional_meshes=excluded_optional_meshes, - expected_proven_root_rigid_bake=proven_root_rigid_bake, + prepared["required_equipment"], + expected_animation_count=len(prepared["animation_names"]), + asset_kind=prepared["asset_kind"], + expected_excluded_optional_meshes=prepared["excluded_optional_meshes"], + expected_proven_root_rigid_bake=prepared["proven_root_rigid_bake"], + expected_pivot_only_model=prepared.get("proven_pivot_only_model", False), expected_embedded_model_animation=( - asset_kind == "animated" - and len(animation_names) == 1 - and animation_names[0] == model_name + prepared["asset_kind"] == "animated" + and len(prepared["animation_names"]) == 1 + and prepared["animation_names"][0] == prepared["model_name"] ), ) validated_texture_overrides = _validate_w3d_texture_override_glb( target, - copied, - texture_override_proof, + prepared["copied"], + prepared["texture_override_proof"], ) if validated_texture_overrides is not None: metrics["textureOverrides"] = validated_texture_overrides @@ -4086,17 +5229,24 @@ def _convert_w3d_bundle( metrics["metrics"]["textureOverrideCount"] = len( validated_texture_overrides["entries"] ) + secondary_skin_proof = prepared["secondary_skin_proof"] if secondary_skin_proof is not None: metrics["secondarySkinStreams"] = secondary_skin_proof + retained = secondary_skin_proof.get("retained") is True metrics["capabilities"][ "secondarySkinStreamsProvenEquivalentAndRemoved" - ] = True + ] = not retained + if retained: + metrics["capabilities"][ + "secondarySkinStreamsRetainedWithUnprovenRedundancy" + ] = True metrics["metrics"]["secondarySkinTransformedMeshCount"] = ( secondary_skin_proof["transformedMeshCount"] ) metrics["metrics"]["secondarySkinRemovedByteCount"] = secondary_skin_proof[ "removedByteCount" ] + no_motion_proof = prepared["no_motion_proof"] if no_motion_proof is not None: metrics["noMotionAnimations"] = no_motion_proof metrics["capabilities"]["headerOnlyNoMotionAnimationsProvenAndRemoved"] = ( @@ -4108,12 +5258,64 @@ def _convert_w3d_bundle( metrics["metrics"]["noMotionRemovedByteCount"] = no_motion_proof[ "removedByteCount" ] - metrics_path = _safe_output(pack_root, report_relative_path) + empty_placeholders = list(prepared.get("empty_placeholder_animations") or []) + if empty_placeholders: + metrics["emptyPlaceholderAnimations"] = empty_placeholders + metrics["capabilities"]["zeroByteRetailAnimationPlaceholdersExcluded"] = True + metrics["metrics"]["emptyPlaceholderAnimationCount"] = len( + empty_placeholders + ) + metrics_path = _safe_output(pack_root, prepared["report_relative_path"]) write_json_atomic(metrics_path, metrics) if not cache_hit: - self._populate_w3d_cache(cache_key, target, combined_log) + with self._w3d_cache_lock(str(prepared["cache_key"])): + self._populate_w3d_cache( + prepared["cache_key"], target, combined_log + ) return [target, metrics_path] + def _convert_w3d_bundle( + self, + staging_sources: list[Path], + output: str | None, + options: dict[str, Any], + pack_root: Path, + profile_id: str, + asset_id: str, + asset_kind: str, + ) -> list[Path]: + prepared = self._prepare_w3d_bundle_job( + -1, + staging_sources, + output, + options, + pack_root, + profile_id, + asset_id, + asset_kind, + ) + if prepared["cache_hit"]: + return self._finalize_w3d_bundle_job( + prepared, prepared["combined_log"], cache_hit=True + ) + isolated_environment = _isolated_blender_environment( + os.environ, prepared["job_root"] + ) + with self._w3d_cache_lock(prepared["cache_key"]): + # Re-check cache under lock in case a peer filled it. + combined_log = self._copy_w3d_cache_hit( + prepared["cache_key"], prepared["target"] + ) + if combined_log is None: + result = run_checked(prepared["command"], env=isolated_environment) + combined_log = result.stdout + "\n" + result.stderr + cache_hit = False + else: + cache_hit = True + return self._finalize_w3d_bundle_job( + prepared, combined_log, cache_hit=cache_hit + ) + def _write_runtime_data( self, pack_root: Path, runtime_data: dict[str, Any] ) -> None: @@ -4140,7 +5342,9 @@ def _is_git_commit(value: Any) -> bool: ) -def _audit_recipe(recipe: Any, errors: list[str]) -> str: +def _audit_recipe( + recipe: Any, errors: list[str], *, require_clean: bool = False +) -> str: if not isinstance(recipe, dict): errors.append("retail provenance importer_recipe is not an object") return "" @@ -4184,8 +5388,11 @@ def _audit_recipe(recipe: Any, errors: list[str]) -> str: errors.append("importer recipe tree digest disagrees with its file inventory") if not _is_git_commit(recipe.get("git_commit")): errors.append("retail provenance importer recipe has no exact git commit") - if not isinstance(recipe.get("git_worktree_clean"), bool): + clean = recipe.get("git_worktree_clean") + if not isinstance(clean, bool): errors.append("retail provenance importer recipe lacks worktree state") + elif require_clean and clean is not True: + errors.append("retail provenance importer recipe is not from a clean release source") return str(declared or "") @@ -4203,10 +5410,8 @@ def _audit_tool_attestations(tools: Any, profile: str, errors: list[str]) -> int PILLOW_TREE_SHA256, PLUGIN_COMMIT, PLUGIN_SUBMODULE_COMMIT, - PYTHON_BASE_DLL_SHA256, - PYTHON_LAUNCHER_SHA256, - PYTHON_RUNTIME_TREE_SHA256, PYTHON_VERSION, + python_runtime_identity_is_pinned, ) required = {"blender", "ffmpeg", "opensage_w3d_plugin", "pillow", "python"} @@ -4252,9 +5457,7 @@ def _audit_tool_attestations(tools: Any, profile: str, errors: list[str]) -> int if ( not isinstance(python, dict) or python.get("version") != PYTHON_VERSION - or python.get("launcher_sha256") != PYTHON_LAUNCHER_SHA256 - or python.get("base_dll_sha256") != PYTHON_BASE_DLL_SHA256 - or python.get("tree_sha256") != PYTHON_RUNTIME_TREE_SHA256 + or not python_runtime_identity_is_pinned(python) or not isinstance(python.get("file_count"), int) or python.get("file_count", 0) <= 0 or not isinstance(python.get("total_bytes"), int) @@ -4316,7 +5519,9 @@ def _audit_retail_provenance( errors.append("retail provenance must declare redistributable=false") summary["importer_recipe_sha256"] = _audit_recipe( - manifest.get("importer_recipe"), errors + manifest.get("importer_recipe"), + errors, + require_clean=profile == "men-fords-v0", ) summary["tool_attestation_count"] = _audit_tool_attestations( manifest.get("tools"), str(profile), errors @@ -4432,8 +5637,25 @@ def _audit_retail_provenance( return summary -def audit_pack(pack_root: Path | str) -> dict[str, Any]: +def audit_pack(pack_root: Path | str, *, light: bool | None = None) -> dict[str, Any]: + """Audit pack outputs against provenance. + + *light* (or OPENBFME_DEV / OPENBFME_DEV_AUDIT=light): verify path + size only, + skip per-file SHA-256 rehash of the full pack (dev iteration speed). + """ + root = Path(pack_root).expanduser().resolve() + if light is None: + light = os.environ.get("OPENBFME_DEV", "").strip().casefold() in { + "1", + "true", + "yes", + } or os.environ.get("OPENBFME_DEV_AUDIT", "").strip().casefold() in { + "1", + "true", + "yes", + "light", + } manifest_path = root / "provenance" / "manifest.json" errors: list[str] = [] checked = 0 @@ -4508,7 +5730,7 @@ def audit_pack(pack_root: Path | str) -> dict[str, Any]: checked += 1 if target.stat().st_size != item.get("size"): errors.append(f"size mismatch: {relative}") - if sha256_file(target) != item.get("sha256"): + elif not light and sha256_file(target) != item.get("sha256"): errors.append(f"hash mismatch: {relative}") excluded = {"provenance/manifest.json", "provenance/audit.json"} @@ -4571,6 +5793,7 @@ def audit_pack(pack_root: Path | str) -> dict[str, Any]: unique_errors = sorted(set(errors)) return { "valid": not unique_errors, + "light": bool(light), "checked_files": checked, "checked_outputs": len(declared_outputs), "errors": unique_errors, diff --git a/importer/openbfme_importer/profile.py b/importer/openbfme_importer/profile.py index 7202e36..b96cd8b 100644 --- a/importer/openbfme_importer/profile.py +++ b/importer/openbfme_importer/profile.py @@ -42,6 +42,7 @@ "sage-apt-runtime", "retail-unit-rules", "sage-particle-definition", + "sage-scripts", "sage-terrain-materials", } SLUG_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") @@ -62,11 +63,14 @@ W3D_INPUT_RESOURCE_IDS_OPTION = "inputResourceIds" W3D_EXCLUDED_OPTIONAL_MESHES_OPTION = "excludedOptionalMeshes" W3D_PROVEN_ROOT_RIGID_BAKE_OPTION = "provenRootRigidBake" +W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION = "provenPivotOnlyModel" W3D_PROVEN_NO_MOTION_ANIMATIONS_OPTION = "provenNoMotionAnimations" W3D_TEXTURE_OVERRIDES_OPTION = "textureOverrides" +W3D_RETAIL_ABSENT_TEXTURES_OPTION = "retailAbsentTextures" W3D_SOURCE_VARIANT_OF_OPTION = "sourceVariantOf" MAX_W3D_OPTIONAL_MESH_EXCLUSIONS = 64 MAX_W3D_TEXTURE_OVERRIDES = 16 +MAX_W3D_RETAIL_ABSENT_TEXTURES = 16 MAX_W3D_NO_MOTION_ANIMATIONS = 16 W3D_CLEAN_MESH_IDENTIFIER_PATTERN = re.compile( r"^[a-z0-9](?:[a-z0-9_]{0,126}[a-z0-9])?$" @@ -179,6 +183,40 @@ def normalize_w3d_texture_overrides(value: Any) -> list[dict[str, str]]: return normalized +def normalize_retail_absent_textures(value: Any) -> list[str]: + """Validate scanner-recorded retail-absent texture basenames.""" + + if ( + not isinstance(value, list) + or len(value) > MAX_W3D_RETAIL_ABSENT_TEXTURES + or any(not isinstance(basename, str) for basename in value) + ): + raise ValueError( + f"{W3D_RETAIL_ABSENT_TEXTURES_OPTION} must be an array of at most " + f"{MAX_W3D_RETAIL_ABSENT_TEXTURES} strings" + ) + if len(value) != len(set(value)): + raise ValueError(f"{W3D_RETAIL_ABSENT_TEXTURES_OPTION} contains duplicates") + for basename in value: + try: + basename_parts = ( + safe_relative_parts(basename) if isinstance(basename, str) else () + ) + except ValueError: + basename_parts = () + if ( + not isinstance(basename, str) + or not W3D_TEXTURE_BASENAME_PATTERN.fullmatch(basename) + or len(basename_parts) != 1 + or Path(basename).suffix.casefold() not in W3D_TEXTURE_SUFFIXES + ): + raise ValueError( + f"{W3D_RETAIL_ABSENT_TEXTURES_OPTION} must contain only safe " + "supported texture basenames" + ) + return sorted(value, key=str.casefold) + + def normalize_w3d_no_motion_animations(value: Any) -> list[dict[str, Any]]: """Validate exact header-only animation declarations and canonicalize order.""" @@ -596,6 +634,14 @@ def _validate_hierarchical_w3d_options(resource: "ResourceRule") -> None: f"resource {resource.id!r} uses " f"{W3D_PROVEN_NO_MOTION_ANIMATIONS_OPTION} without w3d-hierarchical" ) + if ( + W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION in resource.options + and resource.converter != "w3d-hierarchical" + ): + raise ValueError( + f"resource {resource.id!r} uses " + f"{W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION} without w3d-hierarchical" + ) if resource.converter != "w3d-hierarchical": return model = resource.options.get("model") @@ -619,6 +665,18 @@ def _validate_hierarchical_w3d_options(resource: "ResourceRule") -> None: f"resource {resource.id!r} w3d-hierarchical " f"{W3D_PROVEN_ROOT_RIGID_BAKE_OPTION} must be a boolean" ) + pivot_only = resource.options.get(W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION, False) + if not isinstance(pivot_only, bool): + raise ValueError( + f"resource {resource.id!r} w3d-hierarchical " + f"{W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION} must be a boolean" + ) + if pivot_only and root_rigid_bake: + raise ValueError( + f"resource {resource.id!r} w3d-hierarchical cannot combine " + f"{W3D_PROVEN_PIVOT_ONLY_MODEL_OPTION} and " + f"{W3D_PROVEN_ROOT_RIGID_BAKE_OPTION}" + ) if W3D_PROVEN_NO_MOTION_ANIMATIONS_OPTION in resource.options: resource.options[W3D_PROVEN_NO_MOTION_ANIMATIONS_OPTION] = ( normalize_w3d_no_motion_animations( @@ -777,6 +835,17 @@ def load(cls, path: Path | str) -> "ImportProfile": options[W3D_TEXTURE_OVERRIDES_OPTION] = normalize_w3d_texture_overrides( options[W3D_TEXTURE_OVERRIDES_OPTION] ) + if W3D_RETAIL_ABSENT_TEXTURES_OPTION in options: + if converter not in W3D_DEPENDENCY_CONVERTERS: + raise ValueError( + f"resource {resource_id!r} uses " + f"{W3D_RETAIL_ABSENT_TEXTURES_OPTION} without a W3D bundle converter" + ) + options[W3D_RETAIL_ABSENT_TEXTURES_OPTION] = ( + normalize_retail_absent_textures( + options[W3D_RETAIL_ABSENT_TEXTURES_OPTION] + ) + ) resources.append( ResourceRule( id=resource_id, diff --git a/importer/openbfme_importer/progress.py b/importer/openbfme_importer/progress.py new file mode 100644 index 0000000..b12d6b2 --- /dev/null +++ b/importer/openbfme_importer/progress.py @@ -0,0 +1,303 @@ +"""Lightweight stage progress events for CLI and desktop GUI consumers. + +Design: +- Overall ETA is driven by a known stage plan (weights), not by overwriting a + single global unit counter mid-run. +- Within a stage, optional unit counts give finer ETA for that stage only. +- Progress is fail-open: never abort conversion. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import json +import os +import sys +import threading +import time +from pathlib import Path +from typing import Any, Callable, Mapping + + +_LOCK = threading.Lock() +_SINK: Path | None = None +_STARTED = 0.0 +_STAGE = "" +_STAGE_STARTED = 0.0 +_DONE_STAGES: list[str] = [] +_STAGE_PLAN: list[str] = [] +_NEXT_STAGES: list[str] = [] +# Stage-local work units (reset when entering a stage that sets total_units). +_STAGE_TOTAL_UNITS = 0 +_STAGE_DONE_UNITS = 0 +# Optional overall unit budget if caller wants a single counter for the whole run. +_RUN_TOTAL_UNITS = 0 +_RUN_DONE_UNITS = 0 +_LISTENERS: list[Callable[[dict[str, Any]], None]] = [] + + +@dataclass(slots=True) +class ProgressSnapshot: + stage: str + detail: str + elapsed_s: float + stage_elapsed_s: float + done_units: int + total_units: int + done_stages: list[str] = field(default_factory=list) + next_stages: list[str] = field(default_factory=list) + eta_s: float | None = None + fraction: float | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "stage": self.stage, + "detail": self.detail, + "elapsed_s": round(self.elapsed_s, 2), + "stage_elapsed_s": round(self.stage_elapsed_s, 2), + "done_units": self.done_units, + "total_units": self.total_units, + "done_stages": list(self.done_stages), + "next_stages": list(self.next_stages), + "eta_s": None if self.eta_s is None else round(self.eta_s, 1), + "fraction": self.fraction, + "stage_done_units": self.done_units, + "stage_total_units": self.total_units, + } + + +def configure_progress( + *, + sink: Path | str | None = None, + stages: list[str] | None = None, + total_units: int = 0, +) -> None: + """Reset and configure progress for one import run.""" + + global _SINK, _STARTED, _STAGE, _STAGE_STARTED, _DONE_STAGES, _NEXT_STAGES + global _STAGE_PLAN, _STAGE_TOTAL_UNITS, _STAGE_DONE_UNITS + global _RUN_TOTAL_UNITS, _RUN_DONE_UNITS + with _LOCK: + _SINK = Path(sink) if sink else _env_sink() + if _SINK is not None: + _SINK.parent.mkdir(parents=True, exist_ok=True) + _SINK.write_text("", encoding="utf-8") + _STARTED = time.monotonic() + _STAGE = "starting" + _STAGE_STARTED = _STARTED + _DONE_STAGES = [] + _STAGE_PLAN = list(stages or []) + _NEXT_STAGES = list(_STAGE_PLAN) + _STAGE_TOTAL_UNITS = 0 + _STAGE_DONE_UNITS = 0 + _RUN_TOTAL_UNITS = max(0, int(total_units)) + _RUN_DONE_UNITS = 0 + emit("starting", "import run configured", unit_delta=0) + + +def _env_sink() -> Path | None: + raw = os.environ.get("OPENBFME_PROGRESS_FILE", "").strip() + return Path(raw) if raw else None + + +def add_listener(callback: Callable[[dict[str, Any]], None]) -> None: + with _LOCK: + _LISTENERS.append(callback) + + +def clear_listeners() -> None: + with _LOCK: + _LISTENERS.clear() + + +def set_stage_plan(stages: list[str]) -> None: + global _STAGE_PLAN, _NEXT_STAGES + with _LOCK: + _STAGE_PLAN = list(stages) + _NEXT_STAGES = [item for item in stages if item not in _DONE_STAGES] + + +def set_total_units(total: int) -> None: + """Set stage-local total units (does not clobber overall stage-plan ETA).""" + + global _STAGE_TOTAL_UNITS, _STAGE_DONE_UNITS + with _LOCK: + _STAGE_TOTAL_UNITS = max(0, int(total)) + _STAGE_DONE_UNITS = 0 + + +def _compute_eta( + *, + elapsed: float, + stage_elapsed: float, +) -> tuple[float | None, float | None]: + """Return (eta_s, fraction) using stage plan + optional stage units.""" + + fraction: float | None = None + eta_s: float | None = None + + # Prefer known stage plan for overall progress (stable across stages). + plan = _STAGE_PLAN + if plan: + current = _STAGE + partial = 0.0 + if current == "complete": + completed = len(plan) + elif current in plan: + completed = plan.index(current) + if _STAGE_TOTAL_UNITS > 0: + partial = min(1.0, _STAGE_DONE_UNITS / _STAGE_TOTAL_UNITS) + else: + # Unknown/extra stage (e.g. "starting"): count finished plan stages only. + completed = sum(1 for name in plan if name in _DONE_STAGES) + fraction = min(1.0, (completed + partial) / max(1, len(plan))) + if 0.0 < fraction < 1.0 and elapsed > 0: + eta_s = (elapsed / fraction) - elapsed + elif fraction >= 1.0: + eta_s = 0.0 + return eta_s, fraction + + # Fallback: run-level units if provided. + if _RUN_TOTAL_UNITS > 0: + fraction = min(1.0, _RUN_DONE_UNITS / _RUN_TOTAL_UNITS) + if _RUN_DONE_UNITS > 0 and fraction < 1.0 and elapsed > 0: + eta_s = (elapsed / _RUN_DONE_UNITS) * (_RUN_TOTAL_UNITS - _RUN_DONE_UNITS) + return eta_s, fraction + + # Last resort: stage-local units only. + if _STAGE_TOTAL_UNITS > 0: + fraction = min(1.0, _STAGE_DONE_UNITS / _STAGE_TOTAL_UNITS) + if _STAGE_DONE_UNITS > 0 and fraction < 1.0 and stage_elapsed > 0: + eta_s = (stage_elapsed / _STAGE_DONE_UNITS) * ( + _STAGE_TOTAL_UNITS - _STAGE_DONE_UNITS + ) + return eta_s, fraction + + return None, None + + +def emit( + stage: str = "", + detail: str = "", + *, + unit_delta: int = 0, + total_units: int | None = None, + extra: Mapping[str, Any] | None = None, +) -> None: + """Publish one progress event. + + Pass ``stage=""`` to update detail/units without switching stages (safe for + concurrent workers). ``total_units`` sets the *current stage* work budget. + """ + + global _STAGE, _STAGE_STARTED, _STAGE_DONE_UNITS, _STAGE_TOTAL_UNITS + global _STARTED, _NEXT_STAGES, _RUN_DONE_UNITS + now = time.monotonic() + with _LOCK: + if _STARTED <= 0: + _STARTED = now + _STAGE_STARTED = now + if stage and stage != _STAGE: + if _STAGE and _STAGE not in {"", "starting"} and _STAGE not in _DONE_STAGES: + _DONE_STAGES.append(_STAGE) + _STAGE = stage + _STAGE_STARTED = now + _STAGE_DONE_UNITS = 0 + _STAGE_TOTAL_UNITS = 0 + if stage in _NEXT_STAGES: + _NEXT_STAGES = [item for item in _NEXT_STAGES if item != stage] + if total_units is not None: + _STAGE_TOTAL_UNITS = max(0, int(total_units)) + # Setting a stage budget does not imply prior units completed. + if unit_delta == 0: + _STAGE_DONE_UNITS = 0 + if unit_delta: + _STAGE_DONE_UNITS = max(0, _STAGE_DONE_UNITS + int(unit_delta)) + if _RUN_TOTAL_UNITS > 0: + _RUN_DONE_UNITS = max(0, _RUN_DONE_UNITS + int(unit_delta)) + elapsed = now - _STARTED + stage_elapsed = now - _STAGE_STARTED + eta_s, fraction = _compute_eta(elapsed=elapsed, stage_elapsed=stage_elapsed) + next_stages = list(_NEXT_STAGES) + event = { + "ts": time.time(), + "stage": _STAGE, + "detail": detail, + "elapsed_s": round(elapsed, 2), + "stage_elapsed_s": round(stage_elapsed, 2), + "done_units": _STAGE_DONE_UNITS, + "total_units": _STAGE_TOTAL_UNITS, + "run_done_units": _RUN_DONE_UNITS, + "run_total_units": _RUN_TOTAL_UNITS, + "done_stages": list(_DONE_STAGES), + "next_stages": next_stages, + "stage_plan": list(_STAGE_PLAN), + "eta_s": None if eta_s is None else round(max(0.0, eta_s), 1), + "fraction": None if fraction is None else round(fraction, 4), + } + if extra: + event["extra"] = dict(extra) + sink = _SINK + listeners = list(_LISTENERS) + + if sink is not None: + try: + with sink.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, sort_keys=True) + "\n") + except OSError: + pass + + eta_text = ( + f"eta {int(event['eta_s'])}s" if event["eta_s"] is not None else "eta ?" + ) + frac_text = "" + if event["fraction"] is not None: + frac_text = f" | {int(float(event['fraction']) * 100)}%" + units_text = "" + if _STAGE_TOTAL_UNITS > 0: + units_text = f" | units {_STAGE_DONE_UNITS}/{_STAGE_TOTAL_UNITS}" + skipped_text = "" + if extra and extra.get("skipped"): + skipped_text = " | skipped" + line = ( + f"[progress] {event['stage']}: {detail or '…'} | " + f"elapsed {int(elapsed)}s | {eta_text}{frac_text}{units_text}{skipped_text}" + ) + if next_stages: + line += f" | next: {next_stages[0]}" + try: + print(line, file=sys.stderr, flush=True) + except OSError: + pass + + for listener in listeners: + try: + listener(event) + except Exception: + pass + + +def snapshot() -> ProgressSnapshot: + now = time.monotonic() + with _LOCK: + started = _STARTED if _STARTED > 0 else now + stage_started = _STAGE_STARTED if _STAGE_STARTED > 0 else now + elapsed = now - started + stage_elapsed = now - stage_started + eta_s, fraction = _compute_eta(elapsed=elapsed, stage_elapsed=stage_elapsed) + return ProgressSnapshot( + stage=_STAGE or "idle", + detail="", + elapsed_s=elapsed, + stage_elapsed_s=stage_elapsed, + done_units=_STAGE_DONE_UNITS, + total_units=_STAGE_TOTAL_UNITS, + done_stages=list(_DONE_STAGES), + next_stages=list(_NEXT_STAGES), + eta_s=eta_s, + fraction=fraction, + ) + + +def complete(detail: str = "done") -> None: + emit("complete", detail) diff --git a/importer/openbfme_importer/tools.py b/importer/openbfme_importer/tools.py index 00124da..eac8fb9 100644 --- a/importer/openbfme_importer/tools.py +++ b/importer/openbfme_importer/tools.py @@ -70,8 +70,18 @@ def discover_executable(name: str, env_name: str | None = None) -> Path | None: return Path(found).resolve() if found else None -def inspect_tool(name: str, env_name: str | None = None, version_args: Sequence[str] = ("-version",)) -> ToolInfo: - executable = discover_executable(name, env_name) +def inspect_tool( + name: str, + env_name: str | None = None, + version_args: Sequence[str] = ("-version",), + *, + executable: Path | None = None, +) -> ToolInfo: + executable = ( + Path(executable).expanduser().resolve() + if executable is not None + else discover_executable(name, env_name) + ) if not executable: return ToolInfo(name, None, None) try: @@ -111,6 +121,33 @@ def git_revision(repository: Path, relative: str | None = None) -> str | None: return result.stdout.strip().casefold() if result.returncode == 0 else None +def git_revision_at_exact_root(repository: Path) -> str | None: + """Return HEAD only when repository itself is the Git top-level.""" + + git = shutil.which("git") + root = repository.expanduser().resolve() + if not git: + return None + try: + top = subprocess.run( + [git, "rev-parse", "--show-toplevel"], + cwd=root, + capture_output=True, + text=True, + check=False, + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired): + return None + if top.returncode != 0: + return None + try: + discovered = Path(top.stdout.strip()).resolve() + except (OSError, ValueError): + return None + return git_revision(root) if discovered == root else None + + def git_worktree_clean(repository: Path) -> bool: git = shutil.which("git") if not git: diff --git a/importer/openbfme_importer/w3d_input_stage.py b/importer/openbfme_importer/w3d_input_stage.py index 089636f..e2d5328 100644 --- a/importer/openbfme_importer/w3d_input_stage.py +++ b/importer/openbfme_importer/w3d_input_stage.py @@ -366,7 +366,9 @@ def _resolve_output_root(value: Path | str, source_root: Path) -> Path: ) if os.path.lexists(output): if _is_link_like(output): - raise W3DInputStageError("W3D input stage root must not be linked") + raise W3DInputStageError( + "W3D input stage root is linked and must not be linked" + ) if not output.is_dir(): raise W3DInputStageError("W3D input stage root is not a directory") return output diff --git a/importer/requirements-release-win.txt b/importer/requirements-release-win.txt new file mode 100644 index 0000000..7fb2c90 --- /dev/null +++ b/importer/requirements-release-win.txt @@ -0,0 +1,3 @@ +Pillow==12.2.0 --hash=sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5 +fonttools==4.61.1 --hash=sha256:497c31ce314219888c0e2fce5ad9178ca83fe5230b01a5006726cdf3ac9f24d9 +defusedxml==0.7.1 --hash=sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61 diff --git a/importer/tests/test_big.py b/importer/tests/test_big.py index 104f08a..b731d84 100644 --- a/importer/tests/test_big.py +++ b/importer/tests/test_big.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import struct import tempfile from pathlib import Path @@ -96,8 +97,16 @@ def test_cached_bytes_are_verified_against_current_archive(self) -> None: changed[-1] = ord("B") path.write_bytes(changed) changed_archive = BigArchive.open(path) - with self.assertRaises(FileExistsError): - changed_archive.extract(changed_archive.entries, root / "out") + previous = os.environ.get("OPENBFME_EXTRACT_VERIFY") + os.environ["OPENBFME_EXTRACT_VERIFY"] = "full" + try: + with self.assertRaises(FileExistsError): + changed_archive.extract(changed_archive.entries, root / "out") + finally: + if previous is None: + os.environ.pop("OPENBFME_EXTRACT_VERIFY", None) + else: + os.environ["OPENBFME_EXTRACT_VERIFY"] = previous def test_rejects_case_colliding_extraction_targets(self) -> None: with tempfile.TemporaryDirectory() as raw: diff --git a/importer/tests/test_catalog_profile.py b/importer/tests/test_catalog_profile.py index 36e671d..c004233 100644 --- a/importer/tests/test_catalog_profile.py +++ b/importer/tests/test_catalog_profile.py @@ -101,7 +101,32 @@ def test_policy_catalog_ignores_extras_and_rejects_member_payload_drift(self) -> payload[-1] ^= 1 base.write_bytes(payload) os.utime(base, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns)) - self.assertIn("changed archive payload: INI.big", catalog.stale_reasons()) + # Payload sample canary catches bit-flips without full MD5. + reasons = catalog.stale_reasons() + self.assertTrue( + any( + reason.startswith("changed archive payload sample: INI.big") + or reason.startswith("changed archive payload: INI.big") + for reason in reasons + ), + reasons, + ) + # Deep path still reports full policy MD5 mismatch. + previous = os.environ.get("OPENBFME_CATALOG_DEEP") + os.environ["OPENBFME_CATALOG_DEEP"] = "1" + try: + deep_reasons = catalog.stale_reasons() + self.assertTrue( + any( + "changed archive payload" in reason for reason in deep_reasons + ), + deep_reasons, + ) + finally: + if previous is None: + os.environ.pop("OPENBFME_CATALOG_DEEP", None) + else: + os.environ["OPENBFME_CATALOG_DEEP"] = previous with self.assertRaisesRegex(ValueError, "digest changed"): InstallCatalog.build(root, source_policy=policy) @@ -268,9 +293,33 @@ def test_catalog_detects_new_archive_and_same_metadata_directory_change(self) -> base.touch() os.utime(base, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns)) self.assertEqual(base.stat().st_size, original_stat.st_size) + reasons = catalog.stale_reasons() + # Sample canary or deep directory re-parse both detect directory edits. self.assertTrue( - any(reason.startswith("changed archive directory:") for reason in catalog.stale_reasons()) + any( + reason.startswith("changed archive payload sample:") + or reason.startswith("changed archive directory:") + for reason in reasons + ), + reasons, ) + previous = os.environ.get("OPENBFME_CATALOG_DEEP") + os.environ["OPENBFME_CATALOG_DEEP"] = "1" + try: + deep_reasons = catalog.stale_reasons() + self.assertTrue( + any( + reason.startswith("changed archive payload sample:") + or reason.startswith("changed archive directory:") + for reason in deep_reasons + ), + deep_reasons, + ) + finally: + if previous is None: + os.environ.pop("OPENBFME_CATALOG_DEEP", None) + else: + os.environ["OPENBFME_CATALOG_DEEP"] = previous def test_loaded_catalog_entries_are_bound_to_reopened_big_directory(self) -> None: with tempfile.TemporaryDirectory() as raw: diff --git a/importer/tests/test_full_asset_extract.py b/importer/tests/test_full_asset_extract.py index 41fa3a3..0d638c1 100644 --- a/importer/tests/test_full_asset_extract.py +++ b/importer/tests/test_full_asset_extract.py @@ -192,9 +192,23 @@ def test_noop_verifies_cached_bytes_against_archive_payload(self) -> None: archive_path, ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns), ) - self.assertEqual(catalog.stale_reasons(), []) - with self.assertRaisesRegex(RuntimeError, "use --force"): - pipeline.extract_all_assets() + # Payload sample canary flags the archive even when mtime is pinned. + self.assertTrue( + any( + "payload sample" in reason + for reason in catalog.stale_reasons() + ) + ) + previous = os.environ.get("OPENBFME_EXTRACT_VERIFY") + os.environ["OPENBFME_EXTRACT_VERIFY"] = "full" + try: + with self.assertRaisesRegex(RuntimeError, "catalog is stale|use --force"): + pipeline.extract_all_assets() + finally: + if previous is None: + os.environ.pop("OPENBFME_EXTRACT_VERIFY", None) + else: + os.environ["OPENBFME_EXTRACT_VERIFY"] = previous def test_rejects_stale_unsafe_and_over_limit_catalogs_before_writing(self) -> None: with tempfile.TemporaryDirectory() as raw: diff --git a/importer/tests/test_integrity.py b/importer/tests/test_integrity.py index 1f9d86a..2909d02 100644 --- a/importer/tests/test_integrity.py +++ b/importer/tests/test_integrity.py @@ -178,7 +178,7 @@ def test_fords_semantic_contract_binds_sources_recipe_and_tools(self) -> None: "tree_sha256": recipe_digest, "files": [recipe_file], "git_commit": "e" * 40, - "git_worktree_clean": False, + "git_worktree_clean": True, }, "source_game": "bfme2-retail-user-owned", "source_archives": archives, diff --git a/importer/tests/test_release_bundle_contracts.py b/importer/tests/test_release_bundle_contracts.py new file mode 100644 index 0000000..b254fe2 --- /dev/null +++ b/importer/tests/test_release_bundle_contracts.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from openbfme_importer.catalog import InstallCatalog +from openbfme_importer.pipeline import ImportPipeline, _importer_recipe_report + + +class ReleaseBundleContractTests(unittest.TestCase): + def test_custom_state_ffmpeg_precedes_machine_path(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + install = root / "install" + install.mkdir() + state = root / "state" + pinned = state / "tools" / "ffmpeg-8.1.1" / "bin" / "ffmpeg.exe" + pinned.parent.mkdir(parents=True) + pinned.write_bytes(b"pinned") + machine = root / "machine-ffmpeg.exe" + machine.write_bytes(b"machine") + pipeline = ImportPipeline(InstallCatalog(install, (), ()), state) + + with ( + mock.patch.dict(os.environ, {}, clear=False), + mock.patch( + "openbfme_importer.pipeline.discover_executable", + return_value=machine, + ) as discover, + ): + os.environ.pop("OPENBFME_FFMPEG", None) + self.assertEqual(pipeline._ffmpeg_executable(), pinned.resolve()) + + discover.assert_not_called() + + def test_bundled_recipe_uses_validated_release_identity(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + (root / "importer" / "openbfme_importer").mkdir(parents=True) + (root / "importer" / "openbfme_importer" / "entry.py").write_text( + "# fixture\n", encoding="utf-8" + ) + (root / "release-identity.json").write_text( + json.dumps( + { + "schema": "openbfme.bundled-source-identity", + "schemaVersion": 1, + "commit": "a" * 40, + "sourceClean": True, + } + ), + encoding="utf-8", + ) + with mock.patch( + "openbfme_importer.pipeline.repo_root_from_module", + return_value=root, + ): + report = _importer_recipe_report() + + self.assertEqual(report["git_commit"], "a" * 40) + self.assertTrue(report["git_worktree_clean"]) + + def test_invalid_bundled_release_identity_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + (root / "importer" / "openbfme_importer").mkdir(parents=True) + (root / "importer" / "openbfme_importer" / "entry.py").write_text( + "# fixture\n", encoding="utf-8" + ) + (root / "release-identity.json").write_text( + json.dumps( + { + "schema": "openbfme.bundled-source-identity", + "schemaVersion": 1, + "commit": "a" * 40, + "sourceClean": False, + } + ), + encoding="utf-8", + ) + with mock.patch( + "openbfme_importer.pipeline.repo_root_from_module", + return_value=root, + ): + with self.assertRaisesRegex(RuntimeError, "release identity"): + _importer_recipe_report() + + +if __name__ == "__main__": + unittest.main() diff --git a/importer/tests/test_w3d_batch_to_glb.py b/importer/tests/test_w3d_batch_to_glb.py index 3123528..b47fd16 100644 --- a/importer/tests/test_w3d_batch_to_glb.py +++ b/importer/tests/test_w3d_batch_to_glb.py @@ -1034,6 +1034,8 @@ def test_single_job_main_keeps_established_marker_and_report(self) -> None: required_equipment=[], excluded_optional_meshes=[], proven_root_rigid_bake=False, + proven_pivot_only_model=False, + retail_absent_textures=[], output=root / "output.glb", ) adapter.initialize_w3d_converter = lambda _root: calls.__setitem__( @@ -1081,8 +1083,10 @@ def test_plugin_factory_reset_and_shim_are_installed_once(self) -> None: fake_bpy.ops.wm.read_factory_settings = lambda **_kwargs: calls.__setitem__( "factory", calls["factory"] + 1 ) + updater = types.SimpleNamespace(stage_path=None) plugin = types.SimpleNamespace( - register=lambda: calls.__setitem__("register", calls["register"] + 1) + register=lambda: calls.__setitem__("register", calls["register"] + 1), + addon_updater_ops=types.SimpleNamespace(updater=updater), ) sys.modules["io_mesh_w3d"] = plugin adapter.install_shader_material_compatibility_shim = lambda: ( @@ -1105,6 +1109,10 @@ def test_plugin_factory_reset_and_shim_are_installed_once(self) -> None: adapter.initialize_w3d_converter(plugin_root) adapter.initialize_w3d_converter(plugin_root) self.assertEqual(calls, {"factory": 1, "register": 1, "shim": 1}) + self.assertTrue(Path(updater.stage_path).is_dir()) + self.assertFalse( + Path(updater.stage_path).is_relative_to(plugin_root) + ) with self.assertRaisesRegex(RuntimeError, "switch plugin roots"): adapter.initialize_w3d_converter(other_root) finally: diff --git a/importer/tests/test_w3d_converter_phase_evidence.py b/importer/tests/test_w3d_converter_phase_evidence.py index 41f5ddf..6bb1f5d 100644 --- a/importer/tests/test_w3d_converter_phase_evidence.py +++ b/importer/tests/test_w3d_converter_phase_evidence.py @@ -142,6 +142,7 @@ def load_converter_module(): "mesh-box-ambiguity-validation", "mesh-equipment-classification", "required-equipment-validation", + "skin-validation", "render-proof", "animation-import", "scene-validation", @@ -317,6 +318,13 @@ def __init__(self) -> None: self.validate_request = mock.Mock(wraps=CONVERTER.validate_asset_kind_request) self.find_static_rig = mock.Mock(return_value=None) self.find_single_rig = mock.Mock(return_value=None) + self.find_model_rig = mock.Mock( + side_effect=lambda asset_kind: ( + self.find_static_rig() + if asset_kind == "static" + else self.find_single_rig() + ) + ) self.assert_non_animated = mock.Mock() self.remove_geometry = mock.Mock(return_value=[]) @@ -372,6 +380,7 @@ def patched(self): "validate_asset_kind_request": self.validate_request, "find_static_rig": self.find_static_rig, "find_single_rig": self.find_single_rig, + "find_model_rig": self.find_model_rig, "assert_non_animated_scene_has_no_actions": self.assert_non_animated, "remove_non_render_geometry": self.remove_geometry, "convert_proven_additive_materials": self.convert_materials, @@ -706,6 +715,10 @@ def test_real_validation_and_restoration_failures_report_new_phases(self) -> Non harness.find_static_rig.return_value = types.SimpleNamespace( data=types.SimpleNamespace(bones=[]) ) + harness.build_inventory.side_effect = lambda *_args, **_kwargs: ( + [{"vertices": 3, "triangles": 1, "skinned": True}], + {}, + ) with tempfile.TemporaryDirectory() as raw: self._invoke_real_failure( harness, @@ -714,6 +727,33 @@ def test_real_validation_and_restoration_failures_report_new_phases(self) -> Non secret=secret, ) + # An empty carrier with no skinned meshes is a legitimate rigid shape: + # the same harness must now convert instead of failing. + harness = StaticConversionHarness() + harness.find_static_rig.return_value = types.SimpleNamespace( + data=types.SimpleNamespace(bones=[]) + ) + checkpoint = TrackingPhaseCheckpoint() + ledger = FakeAnimationOutputLedger() + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + model, output = harness.paths(root) + with harness.patched(): + report = CONVERTER._convert_w3d_job_impl( + model=model, + asset_kind="static", + animations=[], + required_equipment=[], + excluded_optional_meshes=[], + proven_root_rigid_bake=False, + output=output, + animation_output_ledger=ledger, + phase_checkpoint=checkpoint, + ) + self.assertEqual(report["meshes"], 1) + self.assertEqual(report["bones"], 0) + self.assertEqual(checkpoint.phases.count("skin-validation"), 3) + secret = "PRIVATE_ROOT_BAKE" harness = StaticConversionHarness() harness.find_single_rig.return_value = types.SimpleNamespace( diff --git a/importer/tests/test_w3d_pipeline.py b/importer/tests/test_w3d_pipeline.py index e3c4abe..d3c319a 100644 --- a/importer/tests/test_w3d_pipeline.py +++ b/importer/tests/test_w3d_pipeline.py @@ -1167,7 +1167,7 @@ def test_root_rigid_report_requires_exact_profile_opt_in_and_proof(self) -> None "deform_ambiguity_absent": False, } ) - with self.assertRaisesRegex(RuntimeError, "invalid bones"): + with self.assertRaisesRegex(RuntimeError, "skeleton count does not match"): _validated_w3d_metadata(not_applied, [], asset_kind="hierarchical") malformed = root_rigid_report() diff --git a/importer/tests/test_w3d_presentation.py b/importer/tests/test_w3d_presentation.py index b10ea75..60c586f 100644 --- a/importer/tests/test_w3d_presentation.py +++ b/importer/tests/test_w3d_presentation.py @@ -486,6 +486,7 @@ def scene(): rig = types.SimpleNamespace( type="ARMATURE", data=types.SimpleNamespace(bones=[], animation_data=None), + pose=types.SimpleNamespace(bones=[]), parent=None, parent_type="OBJECT", parent_bone="", @@ -521,13 +522,8 @@ def scene(): malformed_cases = ( ( - "parent", - lambda rig, mesh, objects: setattr(mesh, "parent", None), - "rigidly parented", - ), - ( - "parent-type", - lambda rig, mesh, objects: setattr(mesh, "parent_type", "OBJECT"), + "foreign-parent", + lambda rig, mesh, objects: setattr(mesh, "parent", objects[0]), "rigidly parented", ), ( @@ -536,16 +532,18 @@ def scene(): "ambiguous deformation", ), ( - "modifier", + "deforming-modifier", lambda rig, mesh, objects: setattr( - mesh, "modifiers", [types.SimpleNamespace(type="ARMATURE")] + mesh, "modifiers", [types.SimpleNamespace(type="SUBSURF")] ), "ambiguous deformation", ), ( - "non-root-hierarchy", - lambda rig, mesh, objects: rig.data.bones.append(object()), - "carrier is not empty", + "bone-constraints", + lambda rig, mesh, objects: rig.pose.bones.append( + types.SimpleNamespace(constraints=[object()]) + ), + "bone has constraints", ), ) for name, mutate, message in malformed_cases: @@ -564,6 +562,79 @@ def scene(): "hierarchical", True, rig, [mesh], objects ) + def test_proven_root_rigid_bake_accepts_rigid_multi_pivot_carrier(self) -> None: + def scene(): + bones = [ + types.SimpleNamespace(name="ROOT", constraints=[]), + types.SimpleNamespace(name="DOOR", constraints=[]), + ] + rig = types.SimpleNamespace( + type="ARMATURE", + data=types.SimpleNamespace(bones=bones, animation_data=None), + pose=types.SimpleNamespace(bones=bones), + parent=None, + parent_type="OBJECT", + parent_bone="", + modifiers=[], + constraints=[], + animation_data=None, + ) + + def mesh(parent, parent_type, parent_bone, modifiers, offset): + return types.SimpleNamespace( + type="MESH", + data=types.SimpleNamespace(animation_data=None), + parent=parent, + parent_type=parent_type, + parent_bone=parent_bone, + modifiers=modifiers, + vertex_groups=[], + matrix_world=[ + [1.0, 0.0, 0.0, offset], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ], + animation_data=None, + ) + + bone_parented = mesh( + rig, + "BONE", + "DOOR", + [], + 1.0, + ) + carrier_parented = mesh( + rig, + "ARMATURE", + "", + [types.SimpleNamespace(type="ARMATURE")], + 2.0, + ) + unparented = mesh(None, "OBJECT", "", [], 3.0) + meshes = [bone_parented, carrier_parented, unparented] + objects = FakeCollection([*meshes, rig]) + FAKE_BPY.data = types.SimpleNamespace(objects=objects, actions=[]) + return rig, meshes, objects + + rig, meshes, objects = scene() + report = ADAPTER.bake_proven_root_rigid_hierarchy( + "hierarchical", True, rig, meshes, objects + ) + + self.assertTrue(report["applied"]) + self.assertEqual(report["removed_carriers"], 1) + self.assertEqual(report["baked_meshes"], 3) + for item, offset in zip(meshes, (1.0, 2.0, 3.0)): + self.assertIsNone(item.parent) + self.assertEqual(item.parent_type, "OBJECT") + self.assertEqual(item.parent_bone, "") + self.assertEqual(item.modifiers, []) + self.assertEqual(item.matrix_world[0][3], offset) + self.assertNotIn(rig, list(objects)) + self.assertEqual(list(objects), meshes) + def test_hierarchical_scene_rejects_accidental_actions(self) -> None: FAKE_BPY.data = types.SimpleNamespace(objects=[], actions=[object()]) with self.assertRaisesRegex(RuntimeError, "contains animation actions"): diff --git a/importer/tests/test_w3d_to_glb_fixtures.py b/importer/tests/test_w3d_to_glb_fixtures.py index 2425fc6..fa54fa2 100644 --- a/importer/tests/test_w3d_to_glb_fixtures.py +++ b/importer/tests/test_w3d_to_glb_fixtures.py @@ -1077,8 +1077,11 @@ def operation(): self.assertEqual(result, {"FINISHED"}) self.assertIsNone(suppressed) - self.assertEqual(stdout, expected_stdout) - self.assertEqual(stderr, expected_stderr) + # Capture compaction strips only the byte-exact redundant keyframe + # warning class (its count is tracked for the success report); every + # other byte survives failure replay exactly. + self.assertEqual(stdout, b"stdout-before\nstdout-after\n") + self.assertEqual(stderr, b"stderr-after\n") class W3dAnimationGeometryProofTests(unittest.TestCase): @@ -1470,5 +1473,190 @@ def test_object_mesh_data_and_material_replacement_fail_closed(self) -> None: ADAPTER.assert_render_geometry_unchanged(material_proof, [material_owner]) +class W3dAdditiveVertexMaterialTests(unittest.TestCase): + def _material(self): + material = types.SimpleNamespace( + name="fixture_additive_vertex_material", + use_nodes=True, + shader=types.SimpleNamespace(src_blend="1", dest_blend="1"), + ) + principled = FakeNode("BSDF_PRINCIPLED") + principled.inputs["Base Color"].default_value = (0.0, 0.0, 0.0, 1.0) + material.principled = principled + material.node_tree = types.SimpleNamespace( + nodes=FakeNodeCollection([principled]), links=FakeLinks() + ) + return material + + def _mesh(self, material, colors): + attribute_data = FakeColorAttributeData(colors) + attribute = types.SimpleNamespace( + name="DCG_0", data=attribute_data, domain="CORNER", data_type="BYTE_COLOR" + ) + mesh_data = types.SimpleNamespace( + color_attributes=types.SimpleNamespace(active_color=attribute) + ) + mesh = types.SimpleNamespace( + type="MESH", + data=mesh_data, + material_slots=[types.SimpleNamespace(material=material)], + ) + return mesh, attribute_data + + def _scene(self, material, meshes, images=None): + ADAPTER.bpy.data = types.SimpleNamespace( + objects=list(meshes), + materials=[material], + images=list(images or []), + ) + + def test_textureless_additive_material_converts_vertex_colors(self) -> None: + material = self._material() + mesh, attribute_data = self._mesh( + material, [(0.0, 0.0, 0.0, 1.0), (0.8, 0.4, 0.2, 1.0)] + ) + self._scene(material, [mesh]) + + report = ADAPTER.convert_proven_additive_materials([material]) + + self.assertEqual(report["converted_materials"], 1) + converted = list(attribute_data) + self.assertEqual(converted[0], (0.0, 0.0, 0.0, 0.0)) + self.assertAlmostEqual(converted[1][0], 1.0) + self.assertAlmostEqual(converted[1][1], 0.5) + self.assertAlmostEqual(converted[1][2], 0.25) + self.assertAlmostEqual(converted[1][3], 0.8) + base_color = material.principled.inputs["Base Color"].default_value + self.assertEqual(tuple(base_color), (1.0, 1.0, 1.0, 1.0)) + color_links = [ + link + for link in material.node_tree.links + if link.to_socket is material.principled.inputs["Base Color"] + ] + self.assertEqual(len(color_links), 1) + self.assertEqual(color_links[0].from_node.layer_name, "DCG_0") + alpha_links = [ + link + for link in material.node_tree.links + if link.to_socket is material.principled.inputs["Alpha"] + ] + self.assertEqual(len(alpha_links), 1) + self.assertIs(alpha_links[0].from_node, color_links[0].from_node) + + def test_textureless_additive_material_without_colors_uses_constant(self) -> None: + material = self._material() + mesh = types.SimpleNamespace( + type="MESH", + data=types.SimpleNamespace(color_attributes=None), + material_slots=[types.SimpleNamespace(material=material)], + ) + self._scene(material, [mesh]) + + report = ADAPTER.convert_proven_additive_materials([material]) + + self.assertEqual(report["converted_materials"], 1) + base_color = material.principled.inputs["Base Color"].default_value + self.assertEqual(tuple(base_color), (0.0, 0.0, 0.0, 1.0)) + self.assertEqual(material.principled.inputs["Alpha"].default_value, 0.0) + + def test_textureless_additive_material_with_nonblack_constant_fails(self) -> None: + material = self._material() + material.principled.inputs["Base Color"].default_value = (0.5, 0.0, 0.0, 1.0) + mesh, _attribute_data = self._mesh(material, [(1.0, 1.0, 1.0, 1.0)]) + self._scene(material, [mesh]) + with self.assertRaisesRegex(RuntimeError, "ambiguous color source"): + ADAPTER.convert_proven_additive_materials([material]) + + def test_textureless_additive_material_shared_mesh_fails(self) -> None: + material = self._material() + other = types.SimpleNamespace(name="other") + mesh, _attribute_data = self._mesh(material, [(1.0, 1.0, 1.0, 1.0)]) + mesh.material_slots.append(types.SimpleNamespace(material=other)) + self._scene(material, [mesh]) + with self.assertRaisesRegex(RuntimeError, "shares its render mesh"): + ADAPTER.convert_proven_additive_materials([material]) + + def test_textureless_additive_material_without_meshes_fails(self) -> None: + material = self._material() + self._scene(material, []) + with self.assertRaisesRegex(RuntimeError, "no render mesh"): + ADAPTER.convert_proven_additive_materials([material]) + + +class FakeNodeCollection(list): + def new(self, node_type: str): + node = FakeNode(node_type) + if node_type == "ShaderNodeVertexColor": + node.outputs = FakeSocketCollection( + {name: FakeSocket(name, node) for name in ("Color", "Alpha")} + ) + self.append(node) + return node + + +class FakeColorAttributeData(list): + def foreach_get(self, attribute: str, buffer: list[float]) -> None: + assert attribute == "color" + buffer[:] = [channel for row in self for channel in row] + + def foreach_set(self, attribute: str, values) -> None: + assert attribute == "color" + flat = list(values) + self[:] = [ + tuple(flat[offset : offset + 4]) for offset in range(0, len(flat), 4) + ] + + +class W3dRetailAbsentTextureTests(unittest.TestCase): + def test_normalize_rejects_unsafe_or_duplicate_basenames(self) -> None: + with self.assertRaises(ValueError): + ADAPTER.normalize_retail_absent_textures(["a.tga", "a.tga"]) + with self.assertRaises(ValueError): + ADAPTER.normalize_retail_absent_textures(["../escape.tga"]) + with self.assertRaises(ValueError): + ADAPTER.normalize_retail_absent_textures(["no_extension"]) + with self.assertRaises(ValueError): + ADAPTER.normalize_retail_absent_textures(["evil.exe"]) + self.assertEqual( + ADAPTER.normalize_retail_absent_textures(["B.tga", "a.dds"]), + ["B.tga", "a.dds"], + ) + + def test_clear_only_tolerated_generated_placeholders(self) -> None: + placeholder = types.SimpleNamespace(name="NBElvnBarx_D_NRM.dds", source="GENERATED") + other = types.SimpleNamespace(name="other_missing.dds", source="GENERATED") + staged = types.SimpleNamespace(name="staged.dds", source="FILE") + image_node = types.SimpleNamespace(type="TEX_IMAGE", image=placeholder) + other_node = types.SimpleNamespace(type="TEX_IMAGE", image=other) + nodes = FakeNodeCollection([image_node, other_node]) + ADAPTER.bpy.data = types.SimpleNamespace( + images=FakeImageCollection([placeholder, other, staged]), + materials=[ + types.SimpleNamespace( + node_tree=types.SimpleNamespace(nodes=nodes, links=[]) + ) + ], + ) + + cleared = ADAPTER.clear_retail_absent_textures(["NBElvnBarx_D_NRM.tga"]) + + self.assertEqual(cleared, ["NBElvnBarx_D_NRM.dds"]) + self.assertNotIn(placeholder, ADAPTER.bpy.data.images) + self.assertIn(other, ADAPTER.bpy.data.images) + self.assertIn(staged, ADAPTER.bpy.data.images) + self.assertNotIn(image_node, nodes) + self.assertIn(other_node, nodes) + + def test_unmatched_tolerated_name_fails_closed(self) -> None: + ADAPTER.bpy.data = types.SimpleNamespace(images=[], materials=[]) + with self.assertRaisesRegex(RuntimeError, "did not match a generated placeholder"): + ADAPTER.clear_retail_absent_textures(["absent.tga"]) + + +class FakeImageCollection(list): + def remove(self, value): + super().remove(value) + + if __name__ == "__main__": unittest.main() diff --git a/launcher/OpenBFME.Launcher.Tests/OpenBFME.Launcher.Tests.csproj b/launcher/OpenBFME.Launcher.Tests/OpenBFME.Launcher.Tests.csproj new file mode 100644 index 0000000..156d4ae --- /dev/null +++ b/launcher/OpenBFME.Launcher.Tests/OpenBFME.Launcher.Tests.csproj @@ -0,0 +1,11 @@ + + + Exe + net10.0-windows + enable + enable + + + + + diff --git a/launcher/OpenBFME.Launcher.Tests/Program.cs b/launcher/OpenBFME.Launcher.Tests/Program.cs new file mode 100644 index 0000000..59c6972 --- /dev/null +++ b/launcher/OpenBFME.Launcher.Tests/Program.cs @@ -0,0 +1,433 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using OpenBFME.Launcher; + +var tests = new (string Name, Func Run)[] +{ + ("options", TestOptions), + ("url policy", TestUrlPolicy), + ("manifest validation", TestManifest), + ("manifest signature", TestManifestSignature), + ("install update rollback", TestInstallAndRollback), + ("version retention", TestVersionRetention), + ("channel and downgrade policy", TestChannelAndDowngrade), + ("zip traversal", TestTraversal), + ("Windows zip aliases", TestWindowsArchiveAliases), + ("hash mismatch", TestHashMismatch), + ("scalar importer progress", TestScalarImporterProgress), + ("diagnostic redaction", TestRedaction), + ("bundle inventory", TestBundleInventory) +}; + +var failed = 0; +foreach (var test in tests) +{ + try { await test.Run(); Console.WriteLine($"PASS {test.Name}"); } + catch (Exception error) { failed++; Console.Error.WriteLine($"FAIL {test.Name}: {error}"); } +} +if (failed != 0) return 1; +Console.WriteLine("LAUNCHER_TESTS_PASS"); +return 0; + +static Task TestOptions() +{ + var root = Path.Combine(Path.GetTempPath(), "openbfme-options"); + var options = LauncherOptions.Parse(new[] + { + "--channel", "playtest", "--install-root", root, + "--import-bfme2", "--bfme2-path", @"F:\BFME2" + }); + Check(options.ImportGame == "bfme2", "flags not parsed"); + Check(options.ManifestUri is null, "non-stable channels require an explicit immutable manifest"); + Throws(() => LauncherOptions.Parse(new[] { "--import-bfme2", "--import-rotwk" })); + var stable = LauncherOptions.Parse(Array.Empty()); + Check(stable.ManifestUri?.AbsoluteUri.Contains("/releases/latest/", StringComparison.Ordinal) == true, + "stable channel did not receive the default update feed"); + return Task.CompletedTask; +} + +static Task TestUrlPolicy() +{ + ReleaseUriPolicy.Validate(new Uri("https://github.com/Ancalgonn/open-bfme-engine/releases/download/v1/game.zip")); + ReleaseUriPolicy.ValidateResponse(new Uri("https://release-assets.githubusercontent.com/github-production-release-asset/game.zip")); + Throws(() => ReleaseUriPolicy.Validate(new Uri("http://github.com/Ancalgonn/open-bfme-engine/a"))); + Throws(() => ReleaseUriPolicy.Validate(new Uri("https://evil.example/game.zip"))); + Throws(() => ReleaseUriPolicy.Validate(new Uri("https://github.com/other/repo/game.zip"))); + Throws(() => ReleaseUriPolicy.Validate( + new Uri("https://github.com/evil/repo/Ancalgonn/payload"))); + Throws(() => ReleaseUriPolicy.Validate( + new Uri("https://release-assets.githubusercontent.com/attacker/game.zip"))); + return Task.CompletedTask; +} + +static Task TestManifest() +{ + var json = """ + { + "schema":"openbfme.release-manifest", + "schemaVersion":1, + "repository":"Ancalgonn/open-bfme-engine", + "version":"0.1.0-alpha.1", + "channel":"playtest", + "commit":"0123456789abcdef0123456789abcdef01234567", + "packages":[{ + "name":"game.zip", + "url":"https://github.com/Ancalgonn/open-bfme-engine/releases/download/v0.1.0-alpha.1/game.zip", + "sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "size":123, + "expandedSize":456, + "kind":"game-windows-x64" + },{ + "name":"launcher.zip", + "url":"https://github.com/Ancalgonn/open-bfme-engine/releases/download/v0.1.0-alpha.1/launcher.zip", + "sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "size":456, + "expandedSize":789, + "kind":"launcher-windows-x64" + }] + } + """; + var manifest = ReleaseManifest.Parse(Encoding.UTF8.GetBytes(json)); + Check(manifest.Version == "0.1.0-alpha.1", "version mismatch"); + return Task.CompletedTask; +} + +static Task TestManifestSignature() +{ + var payload = Encoding.UTF8.GetBytes("{\"release\":1}\n"); + using var key = RSA.Create(2048); + var signature = key.SignData( + payload, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + var text = Encoding.ASCII.GetBytes(Convert.ToBase64String(signature)); + var publicPem = key.ExportSubjectPublicKeyInfoPem(); + // ReadOnlySpan cannot be invoked through reflection; verify the production + // overload's cryptographic behavior with a temporary public-key swap helper. + var helper = typeof(ManifestSignature).GetMethods( + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static) + .Single(item => item.GetParameters().Length == 3); + try + { + helper.CreateDelegate()(payload, text, publicPem); + } + catch (InvalidDataException) + { + throw; + } + text[0] = text[0] == (byte)'A' ? (byte)'B' : (byte)'A'; + Throws(() => helper.CreateDelegate()(payload, text, publicPem)); + var productionField = typeof(ManifestSignature).GetField( + "ProductionPublicKey", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + var productionKey = ((string)productionField.GetRawConstantValue()!).Replace("\r\n", "\n").Trim(); + var repositoryKey = File.ReadAllText(Path.GetFullPath(Path.Combine( + AppContext.BaseDirectory, "..", "..", "..", "..", "..", + "tools", "release", "release-manifest-public.pem"))).Replace("\r\n", "\n").Trim(); + Check(productionKey == repositoryKey, "release public key file drifted from the launcher"); + return Task.CompletedTask; +} + +static async Task TestInstallAndRollback() +{ + var root = NewRoot("install"); + try + { + var first = Package("1.0.0", "0123456789abcdef0123456789abcdef01234567", 70_000); + await new ReleaseInstaller().InstallAsync(first.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(first.BytesFor(item)))); + Check(InstallState.Load(root)?.CurrentVersion == "1.0.0", "first version not selected"); + + var second = Package("1.1.0", "1123456789abcdef0123456789abcdef01234567", 80_000); + await new ReleaseInstaller().InstallAsync(second.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(second.BytesFor(item)))); + var updated = InstallState.Load(root)!; + Check(updated.CurrentVersion == "1.1.0" && updated.PreviousVersion == "1.0.0", "update pointer wrong"); + Check(File.Exists(Path.Combine(root, "launcher-current.json")), "launcher update pointer missing"); + Check(Directory.Exists(Path.Combine(root, "launcher-versions", "1.1.0")), + "launcher version was not installed side-by-side"); + File.WriteAllBytes(Path.Combine(root, "versions", "1.1.0", "OpenBFME.exe"), new byte[80_001]); + await ThrowsAsync(() => new ReleaseInstaller().InstallAsync( + second.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(second.BytesFor(item))))); + File.WriteAllBytes(Path.Combine(root, "versions", "1.1.0", "OpenBFME.exe"), new byte[80_000]); + File.WriteAllBytes( + Path.Combine(root, "launcher-versions", "1.1.0", "OpenBFME.Launcher.exe"), + new byte[70_001]); + await ThrowsAsync(() => new ReleaseInstaller().InstallAsync( + second.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(second.BytesFor(item))))); + File.WriteAllBytes( + Path.Combine(root, "launcher-versions", "1.1.0", "OpenBFME.Launcher.exe"), + new byte[70_000]); + var rolled = ReleaseInstaller.Rollback(root); + Check(rolled.CurrentVersion == "1.0.0" && rolled.PreviousVersion == "1.1.0", "rollback pointer wrong"); + Check(rolled.Commit == first.Manifest.Commit && rolled.PreviousCommit == second.Manifest.Commit, + "rollback commit identity wrong"); + Check(rolled.HighestVersion == "1.1.0" && rolled.HighestCommit == second.Manifest.Commit, + "rollback lost the highest accepted release identity"); + var intermediate = Package("1.0.5", "2123456789abcdef0123456789abcdef01234567", 70_000); + await ThrowsAsync(() => new ReleaseInstaller().InstallAsync( + intermediate.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(intermediate.BytesFor(item))))); + using var launcherState = System.Text.Json.JsonDocument.Parse( + File.ReadAllBytes(Path.Combine(root, "launcher-current.json"))); + Check(launcherState.RootElement.GetProperty("currentVersion").GetString() == "1.0.0", + "launcher rollback pointer wrong"); + } + finally { Directory.Delete(root, true); } +} + +static async Task TestChannelAndDowngrade() +{ + var root = NewRoot("channel"); + try + { + var current = Package("2.0.0", "a123456789abcdef0123456789abcdef01234567", 70_000); + await new ReleaseInstaller().InstallAsync( + current.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(current.BytesFor(item))), + expectedChannel: "playtest"); + var old = Package("1.9.0", "b123456789abcdef0123456789abcdef01234567", 70_000); + await ThrowsAsync(() => new ReleaseInstaller().InstallAsync( + old.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(old.BytesFor(item))), + expectedChannel: "playtest")); + await ThrowsAsync(() => new ReleaseInstaller().InstallAsync( + current.Manifest with { Channel = "nightly" }, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(current.BytesFor(item))), + expectedChannel: "playtest")); + } + finally { Directory.Delete(root, true); } +} + +static async Task TestVersionRetention() +{ + var root = NewRoot("retention"); + try + { + foreach (var (version, commit) in new[] + { + ("1.0.0", "0123456789abcdef0123456789abcdef01234567"), + ("1.1.0", "1123456789abcdef0123456789abcdef01234567"), + ("1.2.0", "2123456789abcdef0123456789abcdef01234567") + }) + { + var package = Package(version, commit, 70_000); + await new ReleaseInstaller().InstallAsync( + package.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(package.BytesFor(item)))); + } + Check(!Directory.Exists(Path.Combine(root, "versions", "1.0.0")), + "obsolete game version was retained"); + Check(!Directory.Exists(Path.Combine(root, "launcher-versions", "1.0.0")), + "obsolete launcher version was retained"); + Check(Directory.Exists(Path.Combine(root, "versions", "1.1.0")) && + Directory.Exists(Path.Combine(root, "versions", "1.2.0")), + "current rollback pair was pruned"); + } + finally { Directory.Delete(root, true); } +} + +static async Task TestTraversal() +{ + var root = NewRoot("traversal"); + try + { + var bytes = Zip(archive => + { + var entry = archive.CreateEntry("../escape.txt"); + using var writer = new StreamWriter(entry.Open()); + writer.Write("escape"); + }); + var package = ManifestFor(bytes, "2.0.0", "2123456789abcdef0123456789abcdef01234567"); + await ThrowsAsync(() => new ReleaseInstaller().InstallAsync( + package.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(package.BytesFor(item))))); + Check(!File.Exists(Path.Combine(root, "escape.txt")), "archive escaped staging"); + } + finally { Directory.Delete(root, true); } +} + +static async Task TestWindowsArchiveAliases() +{ + foreach (var name in new[] { "OpenBFME.exe:payload", "CON.txt", "trailing. " }) + { + var root = NewRoot("windows-path"); + try + { + var bytes = Zip(archive => Write(archive, name, new byte[] { 1 })); + var package = ManifestFor(bytes, "2.0.1", "2123456789abcdef0123456789abcdef01234567"); + await ThrowsAsync(() => new ReleaseInstaller().InstallAsync( + package.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream(package.BytesFor(item))))); + } + finally { Directory.Delete(root, true); } + } +} + +static async Task TestHashMismatch() +{ + var root = NewRoot("hash"); + try + { + var package = Package("3.0.0", "3123456789abcdef0123456789abcdef01234567", 70_000); + var tampered = package.GameBytes.ToArray(); + tampered[^1] ^= 0xff; + await ThrowsAsync(() => new ReleaseInstaller().InstallAsync( + package.Manifest, root, null, default, + (item, _) => Task.FromResult(new MemoryStream( + item.Kind == "game-windows-x64" ? tampered : package.LauncherBytes)))); + Check(InstallState.Load(root) is null, "tampered package was selected"); + } + finally { Directory.Delete(root, true); } +} + +static Task TestRedaction() +{ + var method = typeof(ImporterRunner).GetMethod("Redact", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + var value = (string)method.Invoke(null, new object[] { @"failed at F:\BFME2\data\asset.big" })!; + Check(!value.Contains("F:\\BFME2", StringComparison.OrdinalIgnoreCase), "private path leaked"); + return Task.CompletedTask; +} + +static Task TestScalarImporterProgress() +{ + var method = typeof(ImporterRunner).GetMethod("ParseProgressLine", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + var value = (ImportProgress)method.Invoke(null, new object[] { "\"bootstrap-complete\"" })!; + Check(value.Phase == "Importing", "scalar JSON progress phase changed"); + Check(value.Message.Contains("bootstrap-complete", StringComparison.Ordinal), + "scalar JSON progress was discarded"); + return Task.CompletedTask; +} + +static Task TestBundleInventory() +{ + var root = NewRoot("inventory"); + try + { + Directory.CreateDirectory(Path.Combine(root, "python")); + File.WriteAllText(Path.Combine(root, "python", "python.exe"), "runtime"); + var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes("runtime"))).ToLowerInvariant(); + File.WriteAllText(Path.Combine(root, "openbfme-bundle-inventory.json"), $$""" + {"schema":"openbfme.bundle-inventory","schemaVersion":1,"files":[ + {"path":"python/python.exe","size":7,"sha256":"{{hash}}"} + ]} + """); + var inventory = typeof(ImporterRunner).Assembly.GetType("OpenBFME.Launcher.BundleInventory")!; + var verify = inventory.GetMethod("Verify", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + verify.Invoke(null, new object[] { root }); + File.WriteAllText(Path.Combine(root, "python", "sitecustomize.py"), "raise SystemExit()"); + Throws(() => verify.Invoke(null, new object[] { root })); + } + finally { Directory.Delete(root, true); } + return Task.CompletedTask; +} + +static TestPackageSet Package(string version, string commit, int exeBytes) +{ + var bytes = Zip(archive => + { + Write(archive, "OpenBFME.exe", new byte[exeBytes]); + Write(archive, "OpenBFME.pck", new byte[] { 1, 2, 3 }); + }); + return ManifestFor(bytes, version, commit); +} + +static TestPackageSet ManifestFor(byte[] bytes, string version, string commit) +{ + var sha = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + var launcherBytes = LauncherZip(); + var launcherSha = Convert.ToHexString(SHA256.HashData(launcherBytes)).ToLowerInvariant(); + var manifest = new ReleaseManifest(ReleaseManifest.ExpectedSchema, 1, + "Ancalgonn/open-bfme-engine", version, "playtest", commit, + new[] + { + new ReleasePackage("game.zip", + $"https://github.com/Ancalgonn/open-bfme-engine/releases/download/v{version}/game.zip", + sha, bytes.Length, ExpandedSize(bytes), "game-windows-x64"), + new ReleasePackage("launcher.zip", + $"https://github.com/Ancalgonn/open-bfme-engine/releases/download/v{version}/launcher.zip", + launcherSha, launcherBytes.Length, ExpandedSize(launcherBytes), "launcher-windows-x64") + }); + return new TestPackageSet(manifest, bytes, launcherBytes); +} + +static byte[] LauncherZip() +{ + var executable = new byte[70_000]; + var hash = Convert.ToHexString(SHA256.HashData(executable)).ToLowerInvariant(); + var inventory = Encoding.UTF8.GetBytes($$""" + {"schema":"openbfme.bundle-inventory","schemaVersion":1,"files":[ + {"path":"OpenBFME.Launcher.exe","size":70000,"sha256":"{{hash}}"} + ]} + """); + return Zip(archive => + { + Write(archive, "OpenBFME.Launcher.exe", executable); + Write(archive, "openbfme-bundle-inventory.json", inventory); + }); +} + +static byte[] Zip(Action build) +{ + using var memory = new MemoryStream(); + using (var archive = new ZipArchive(memory, ZipArchiveMode.Create, true)) build(archive); + return memory.ToArray(); +} + +static long ExpandedSize(byte[] bytes) +{ + using var memory = new MemoryStream(bytes); + using var archive = new ZipArchive(memory, ZipArchiveMode.Read); + return archive.Entries.Sum(entry => entry.Length); +} + +static void Write(ZipArchive archive, string name, byte[] bytes) +{ + var entry = archive.CreateEntry(name, CompressionLevel.NoCompression); + using var stream = entry.Open(); + stream.Write(bytes); +} + +static string NewRoot(string suffix) +{ + var root = Path.Combine(Path.GetTempPath(), $"openbfme-launcher-{suffix}-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + return root; +} + +static void Check(bool condition, string message) +{ + if (!condition) throw new InvalidOperationException(message); +} + +static void Throws(Action action) where T : Exception +{ + try { action(); } + catch (T) { return; } + throw new InvalidOperationException($"Expected {typeof(T).Name}."); +} + +static async Task ThrowsAsync(Func action) where T : Exception +{ + try { await action(); } + catch (T) { return; } + throw new InvalidOperationException($"Expected {typeof(T).Name}."); +} + +delegate void ManifestVerifier( + ReadOnlySpan manifest, + ReadOnlySpan signature, + string publicKeyPem); + +sealed record TestPackageSet( + ReleaseManifest Manifest, + byte[] GameBytes, + byte[] LauncherBytes) +{ + internal byte[] BytesFor(ReleasePackage package) => + package.Kind == "game-windows-x64" ? GameBytes : LauncherBytes; +} diff --git a/launcher/OpenBFME.Launcher/App.xaml b/launcher/OpenBFME.Launcher/App.xaml new file mode 100644 index 0000000..a2f0e6a --- /dev/null +++ b/launcher/OpenBFME.Launcher/App.xaml @@ -0,0 +1,17 @@ + + + + + + + diff --git a/launcher/OpenBFME.Launcher/App.xaml.cs b/launcher/OpenBFME.Launcher/App.xaml.cs new file mode 100644 index 0000000..deaee03 --- /dev/null +++ b/launcher/OpenBFME.Launcher/App.xaml.cs @@ -0,0 +1,76 @@ +namespace OpenBFME.Launcher; + +public partial class App : System.Windows.Application +{ + protected override async void OnStartup(System.Windows.StartupEventArgs e) + { + base.OnStartup(e); + LauncherOptions options; + try { options = LauncherOptions.Parse(e.Args); } + catch (Exception error) + { + Console.Error.WriteLine(error.Message); + Shutdown(2); + return; + } + try + { + if (LauncherSelfUpdate.RelaunchSelected(options, e.Args)) + { + Shutdown(0); + return; + } + } + catch (Exception error) + { + Console.Error.WriteLine($"Selected launcher update is invalid: {error.Message}"); + Shutdown(1); + return; + } + if (!options.Headless) + { + new MainWindow().Show(); + return; + } + + ShutdownMode = System.Windows.ShutdownMode.OnExplicitShutdown; + try + { + var service = new LauncherService(options); + if (options.ImportGame is not null && options.RetailPath is not null) + { + var state = Path.Combine(options.InstallRoot, ".private", "retail-work"); + var exit = await service.Importer.RunAsync( + AppContext.BaseDirectory, + options.ImportGame, + options.RetailPath, + state, + null, + CancellationToken.None); + if (exit != 0) throw new InvalidOperationException($"Importer exited with code {exit}."); + } + else if (!options.NoUpdate && options.ManifestUri is not null) + { + var manifest = await service.Installer.FetchManifestAsync( + options.ManifestUri, CancellationToken.None); + await service.Installer.InstallAsync( + manifest, options.InstallRoot, null, CancellationToken.None, + expectedChannel: options.Channel); + } + if (options.VerifyOnly) + { + var current = service.Current + ?? throw new InvalidOperationException("No installed release is selected."); + ReleaseInstaller.VerifyInstalledVersion(Path.Combine( + options.InstallRoot, "versions", current.CurrentVersion), + current.CurrentVersion, current.Commit); + } + Shutdown(0); + } + catch (Exception error) + { + Console.Error.WriteLine(error.Message); + Shutdown(1); + } + } +} diff --git a/launcher/OpenBFME.Launcher/BundleInventory.cs b/launcher/OpenBFME.Launcher/BundleInventory.cs new file mode 100644 index 0000000..4cc446a --- /dev/null +++ b/launcher/OpenBFME.Launcher/BundleInventory.cs @@ -0,0 +1,95 @@ +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace OpenBFME.Launcher; + +internal sealed record BundleInventoryEntry( + [property: JsonPropertyName("path")] string Path, + [property: JsonPropertyName("size")] long Size, + [property: JsonPropertyName("sha256")] string Sha256); + +internal sealed record BundleInventoryDocument( + [property: JsonPropertyName("schema")] string Schema, + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("files")] IReadOnlyList Files); + +internal static class BundleInventory +{ + internal const string FileName = "openbfme-bundle-inventory.json"; + + internal static void Verify(string bundleRoot) + { + var root = Path.GetFullPath(bundleRoot); + var inventoryPath = Path.Combine(root, FileName); + if (!File.Exists(inventoryPath)) + throw new InvalidDataException("Bundled runtime inventory is missing."); + if (new FileInfo(inventoryPath).Length > 4 * 1024 * 1024) + throw new InvalidDataException("Bundled runtime inventory is too large."); + var document = JsonSerializer.Deserialize( + File.ReadAllBytes(inventoryPath), + new JsonSerializerOptions { PropertyNameCaseInsensitive = false }) + ?? throw new InvalidDataException("Bundled runtime inventory is empty."); + if (document.Schema != "openbfme.bundle-inventory" || document.SchemaVersion != 1) + throw new InvalidDataException("Bundled runtime inventory schema is invalid."); + + var expected = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var entry in document.Files) + { + if (!IsSafeRelativePath(entry.Path) || entry.Path.Equals(FileName, StringComparison.OrdinalIgnoreCase) || + entry.Path.Equals(InstalledVersionIdentity.FileName, StringComparison.OrdinalIgnoreCase) || + entry.Size < 0 || !System.Text.RegularExpressions.Regex.IsMatch(entry.Sha256, "^[0-9a-f]{64}$") || + !expected.TryAdd(entry.Path, entry)) + throw new InvalidDataException("Bundled runtime inventory contains an invalid entry."); + } + + var observed = EnumerateContainedFiles(root); + if (observed.Count != expected.Count) + throw new InvalidDataException("Bundled runtime file set does not match its inventory."); + foreach (var pair in observed) + { + if (!expected.TryGetValue(pair.Key, out var entry)) + throw new InvalidDataException("Bundled runtime contains an unlisted file."); + var info = new FileInfo(pair.Value); + if (info.Length != entry.Size) + throw new InvalidDataException("Bundled runtime file size mismatch."); + var actual = SHA256.HashData(File.ReadAllBytes(pair.Value)); + if (!CryptographicOperations.FixedTimeEquals(actual, Convert.FromHexString(entry.Sha256))) + throw new InvalidDataException("Bundled runtime file hash mismatch."); + } + } + + private static Dictionary EnumerateContainedFiles(string root) + { + var files = new Dictionary(StringComparer.OrdinalIgnoreCase); + var pending = new Stack(); + pending.Push(root); + while (pending.Count > 0) + { + var directory = pending.Pop(); + foreach (var path in Directory.EnumerateFileSystemEntries(directory)) + { + var attributes = File.GetAttributes(path); + if ((attributes & FileAttributes.ReparsePoint) != 0) + throw new InvalidDataException("Bundled runtime contains a reparse point."); + if ((attributes & FileAttributes.Directory) != 0) + { + pending.Push(path); + continue; + } + var relative = Path.GetRelativePath(root, path).Replace('\\', '/'); + if (relative.Equals(FileName, StringComparison.OrdinalIgnoreCase) || + relative.Equals(InstalledVersionIdentity.FileName, StringComparison.OrdinalIgnoreCase)) continue; + if (!files.TryAdd(relative, path)) + throw new InvalidDataException("Bundled runtime contains duplicate Windows paths."); + } + } + return files; + } + + private static bool IsSafeRelativePath(string path) => + !string.IsNullOrWhiteSpace(path) && + !path.Contains('\\') && + !Path.IsPathRooted(path) && + path.Split('/').All(segment => segment.Length > 0 && segment is not "." and not ".."); +} diff --git a/launcher/OpenBFME.Launcher/GlobalUsings.cs b/launcher/OpenBFME.Launcher/GlobalUsings.cs new file mode 100644 index 0000000..0f4b1c3 --- /dev/null +++ b/launcher/OpenBFME.Launcher/GlobalUsings.cs @@ -0,0 +1,6 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Linq; +global using System.Threading; +global using System.Threading.Tasks; diff --git a/launcher/OpenBFME.Launcher/ImporterRunner.cs b/launcher/OpenBFME.Launcher/ImporterRunner.cs new file mode 100644 index 0000000..1ba95bd --- /dev/null +++ b/launcher/OpenBFME.Launcher/ImporterRunner.cs @@ -0,0 +1,191 @@ +using System.Diagnostics; +using System.IO; +using System.Text.Json; + +namespace OpenBFME.Launcher; + +public sealed record ImportProgress(string Phase, string Message, double? Percent); + +public sealed class ImporterRunner +{ + public async Task RunAsync( + string launcherDirectory, + string game, + string retailPath, + string stateRoot, + IProgress? progress, + CancellationToken cancellationToken) + { + if (game is not ("bfme2" or "rotwk")) throw new ArgumentOutOfRangeException(nameof(game)); + BundleInventory.Verify(launcherDirectory); + var script = Path.GetFullPath(Path.Combine(launcherDirectory, "tools", "openbfme_import.py")); + if (!File.Exists(script)) throw new FileNotFoundException("Bundled importer entry point is missing.", script); + var retail = Path.GetFullPath(retailPath); + var state = Path.GetFullPath(stateRoot); + var privateRoot = Directory.GetParent(state) + ?? throw new InvalidOperationException("Importer state root has no parent."); + var contentRoot = Path.Combine(privateRoot.FullName, "content-packs"); + var baseProfile = Path.GetFullPath(Path.Combine( + launcherDirectory, "importer", "profiles", "men-fords-v1.json")); + if (!Directory.Exists(retail)) throw new DirectoryNotFoundException("Retail installation is missing."); + if (!File.Exists(baseProfile)) throw new FileNotFoundException("Bundled base import profile is missing.", baseProfile); + Directory.CreateDirectory(state); + + progress?.Report(new ImportProgress("Tools", "Verifying pinned conversion tools.", null)); + var bootstrap = NewProcess(launcherDirectory); + foreach (var argument in new[] { script, "--state-root", state, "--json", "bootstrap-tools" }) + bootstrap.ArgumentList.Add(argument); + using (var bootstrapProcess = new Process { StartInfo = bootstrap, EnableRaisingEvents = true }) + { + bootstrapProcess.Start(); + var bootstrapOut = PumpAsync(bootstrapProcess.StandardOutput, progress, cancellationToken); + var bootstrapErr = PumpErrorsAsync(bootstrapProcess.StandardError, progress, cancellationToken); + await bootstrapProcess.WaitForExitAsync(cancellationToken); + await Task.WhenAll(bootstrapOut, bootstrapErr); + if (bootstrapProcess.ExitCode != 0) return bootstrapProcess.ExitCode; + } + + var start = NewProcess(launcherDirectory); + var firstCommand = game == "bfme2" + ? new[] + { + script, "--state-root", state, "--json", "build", + "--install", retail, "--game", game, "--profile", "men-fords-v0", + "--godot-content-root", contentRoot + } + : new[] + { + script, "--state-root", state, "--json", "import-faction", + "--install", retail, "--game", game, "--faction", "angmar", "--convert" + }; + foreach (var argument in firstCommand) + start.ArgumentList.Add(argument); + + using var process = new Process { StartInfo = start, EnableRaisingEvents = true }; + process.Start(); + var stdout = PumpAsync(process.StandardOutput, progress, cancellationToken); + var stderr = PumpErrorsAsync(process.StandardError, progress, cancellationToken); + await process.WaitForExitAsync(cancellationToken); + await Task.WhenAll(stdout, stderr); + if (process.ExitCode != 0) return process.ExitCode; + if (game == "bfme2") return 0; + + progress?.Report(new ImportProgress("Packaging", "Building and selecting the local content pack.", null)); + var publish = NewProcess(launcherDirectory); + foreach (var argument in new[] + { + script, "--state-root", state, "--json", "publish-faction-to-slice", + "--install", retail, "--game", game, "--faction", game == "rotwk" ? "angmar" : "men", + "--base-profile", baseProfile, "--godot-content-root", contentRoot + }) + publish.ArgumentList.Add(argument); + using var publishProcess = new Process { StartInfo = publish, EnableRaisingEvents = true }; + publishProcess.Start(); + var publishOut = PumpAsync(publishProcess.StandardOutput, progress, cancellationToken); + var publishErr = PumpErrorsAsync(publishProcess.StandardError, progress, cancellationToken); + await publishProcess.WaitForExitAsync(cancellationToken); + await Task.WhenAll(publishOut, publishErr); + return publishProcess.ExitCode; + } + + private static string ResolvePython(string launcherDirectory) + { + var bundled = Path.Combine(launcherDirectory, "python", "python.exe"); + if (!File.Exists(bundled)) + throw new FileNotFoundException("Bundled pinned Python runtime is missing.", bundled); + return bundled; + } + + private static ProcessStartInfo NewProcess(string launcherDirectory) + { + var start = new ProcessStartInfo + { + FileName = ResolvePython(launcherDirectory), + WorkingDirectory = launcherDirectory, + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true + }.WithImporterEnvironment(); + start.ArgumentList.Add("-X"); + start.ArgumentList.Add("utf8"); + start.ArgumentList.Add("-B"); + start.ArgumentList.Add("-I"); + start.ArgumentList.Add("-S"); + return start; + } + + private static async Task PumpAsync( + StreamReader reader, + IProgress? progress, + CancellationToken token) + { + while (await reader.ReadLineAsync(token) is { } line) + { + var parsed = ParseProgressLine(line); + progress?.Report(parsed); + } + } + + internal static ImportProgress ParseProgressLine(string line) + { + if (line.Length > 64 * 1024) + throw new InvalidDataException("Importer progress record is too large."); + try + { + using var json = JsonDocument.Parse(line); + var root = json.RootElement; + if (root.ValueKind != JsonValueKind.Object) + return new ImportProgress("Importing", Redact(line), null); + var phase = root.TryGetProperty("stage", out var p) + ? p.GetString() ?? "Importing" + : "Importing"; + var message = root.TryGetProperty("detail", out var m) + ? m.GetString() ?? phase + : phase; + double? percent = root.TryGetProperty("fraction", out var n) + && n.TryGetDouble(out var value) + ? Math.Clamp(value * 100, 0, 100) + : null; + return new ImportProgress(phase, Redact(message), percent); + } + catch (JsonException) + { + return new ImportProgress("Importing", Redact(line), null); + } + } + + private static async Task PumpErrorsAsync( + StreamReader reader, + IProgress? progress, + CancellationToken token) + { + while (await reader.ReadLineAsync(token) is { } line) + { + var message = Redact(line); + if (progress is null) + Console.Error.WriteLine(message); + else + progress.Report(new ImportProgress("Importer", message, null)); + } + } + + internal static string Redact(string text) + { + var result = text; + foreach (var prefix in new[] { @"[A-Za-z]:\\", @"\\\\[^\\\s]+\\[^\\\s]+\\" }) + result = System.Text.RegularExpressions.Regex.Replace(result, + prefix + @"[^\r\n""]+", "", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + return result.Length <= 400 ? result : result[..400] + "…"; + } +} + +internal static class ProcessStartInfoExtensions +{ + internal static ProcessStartInfo WithImporterEnvironment(this ProcessStartInfo start) + { + start.Environment["PYTHONUTF8"] = "1"; + start.Environment["PYTHONDONTWRITEBYTECODE"] = "1"; + return start; + } +} diff --git a/launcher/OpenBFME.Launcher/InstallState.cs b/launcher/OpenBFME.Launcher/InstallState.cs new file mode 100644 index 0000000..1c4692c --- /dev/null +++ b/launcher/OpenBFME.Launcher/InstallState.cs @@ -0,0 +1,66 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace OpenBFME.Launcher; + +public sealed record InstallState( + [property: JsonPropertyName("schema")] string Schema, + [property: JsonPropertyName("currentVersion")] string CurrentVersion, + [property: JsonPropertyName("previousVersion")] string? PreviousVersion, + [property: JsonPropertyName("commit")] string Commit, + [property: JsonPropertyName("previousCommit")] string? PreviousCommit, + [property: JsonPropertyName("highestVersion")] string HighestVersion, + [property: JsonPropertyName("highestCommit")] string HighestCommit) +{ + public const string ExpectedSchema = "openbfme.install-state"; + + public static InstallState? Load(string root) + { + var path = Path.Combine(root, "current.json"); + if (!File.Exists(path)) return null; + var state = JsonSerializer.Deserialize(File.ReadAllBytes(path)) + ?? throw new InvalidDataException("Install state is empty."); + state.Validate(); + return state; + } + + public static void SaveAtomic(string root, InstallState state) + { + state.Validate(); + Directory.CreateDirectory(root); + var path = Path.Combine(root, "current.json"); + var temporary = path + $".{Environment.ProcessId}.tmp"; + var bytes = JsonSerializer.SerializeToUtf8Bytes(state, + new JsonSerializerOptions { WriteIndented = true }); + using (var stream = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write, + FileShare.None, 4096, FileOptions.WriteThrough)) + { + stream.Write(bytes); + stream.WriteByte((byte)'\n'); + stream.Flush(true); + } + if (File.Exists(path)) File.Replace(temporary, path, null); + else File.Move(temporary, path); + } + + private void Validate() + { + var highestMatchesCurrent = HighestVersion == CurrentVersion && HighestCommit == Commit; + var highestMatchesPrevious = HighestVersion == PreviousVersion && HighestCommit == PreviousCommit; + if (Schema != ExpectedSchema || + !SafeVersion(CurrentVersion) || + !FullSha1(Commit) || + !SafeVersion(HighestVersion) || + !FullSha1(HighestCommit) || + !(highestMatchesCurrent || highestMatchesPrevious) || + (PreviousVersion is null) != (PreviousCommit is null) || + PreviousVersion is not null && (!SafeVersion(PreviousVersion) || !FullSha1(PreviousCommit!)) || + PreviousVersion == CurrentVersion) + throw new InvalidDataException("Install state is invalid."); + } + + private static bool SafeVersion(string value) => + System.Text.RegularExpressions.Regex.IsMatch(value, "^[0-9A-Za-z][0-9A-Za-z._-]{0,63}$"); + private static bool FullSha1(string value) => + System.Text.RegularExpressions.Regex.IsMatch(value, "^[0-9a-f]{40}$"); +} diff --git a/launcher/OpenBFME.Launcher/InstalledVersionIdentity.cs b/launcher/OpenBFME.Launcher/InstalledVersionIdentity.cs new file mode 100644 index 0000000..7217168 --- /dev/null +++ b/launcher/OpenBFME.Launcher/InstalledVersionIdentity.cs @@ -0,0 +1,116 @@ +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace OpenBFME.Launcher; + +internal sealed record InstalledFileIdentity( + [property: JsonPropertyName("path")] string Path, + [property: JsonPropertyName("size")] long Size, + [property: JsonPropertyName("sha256")] string Sha256); + +internal sealed record InstalledVersionIdentity( + [property: JsonPropertyName("schema")] string Schema, + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("version")] string Version, + [property: JsonPropertyName("commit")] string Commit, + [property: JsonPropertyName("packageSha256")] string PackageSha256, + [property: JsonPropertyName("files")] IReadOnlyList Files) +{ + internal const string FileName = ".openbfme-version.json"; + private const string ExpectedSchema = "openbfme.installed-version"; + + internal static void Write(string root, ReleaseManifest manifest, ReleasePackage package) + { + var rows = Inventory(root).Select(pair => + { + var info = new FileInfo(pair.Value); + return new InstalledFileIdentity( + pair.Key, + info.Length, + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(pair.Value))).ToLowerInvariant()); + }).ToArray(); + var identity = new InstalledVersionIdentity( + ExpectedSchema, 1, manifest.Version, manifest.Commit, package.Sha256, rows); + var path = Path.Combine(root, FileName); + File.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(identity, + new JsonSerializerOptions { WriteIndented = true })); + } + + internal static void Verify( + string root, + string? expectedVersion = null, + string? expectedCommit = null, + string? expectedPackageSha256 = null) + { + var path = Path.Combine(root, FileName); + if (!File.Exists(path)) throw new InvalidDataException("Installed version identity is missing."); + var identity = JsonSerializer.Deserialize(File.ReadAllBytes(path), + new JsonSerializerOptions { PropertyNameCaseInsensitive = false }) + ?? throw new InvalidDataException("Installed version identity is empty."); + if (identity.Schema != ExpectedSchema || identity.SchemaVersion != 1 || + !SafeVersion(identity.Version) || !FullSha1(identity.Commit) || !Sha256(identity.PackageSha256)) + throw new InvalidDataException("Installed version identity is invalid."); + if (expectedVersion is not null && identity.Version != expectedVersion || + expectedCommit is not null && identity.Commit != expectedCommit || + expectedPackageSha256 is not null && identity.PackageSha256 != expectedPackageSha256) + throw new InvalidDataException("Installed version identity does not match the selected release."); + + var expected = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var entry in identity.Files) + if (!SafePath(entry.Path) || entry.Size < 0 || !Sha256(entry.Sha256) || + !expected.TryAdd(entry.Path, entry)) + throw new InvalidDataException("Installed file identity is invalid."); + var observed = Inventory(root); + if (observed.Count != expected.Count) + throw new InvalidDataException("Installed version file set changed."); + foreach (var pair in observed) + { + if (!expected.TryGetValue(pair.Key, out var entry)) + throw new InvalidDataException("Installed version contains an unlisted file."); + var info = new FileInfo(pair.Value); + if (info.Length != entry.Size) + throw new InvalidDataException("Installed version file size changed."); + var hash = SHA256.HashData(File.ReadAllBytes(pair.Value)); + if (!CryptographicOperations.FixedTimeEquals(hash, Convert.FromHexString(entry.Sha256))) + throw new InvalidDataException("Installed version file hash changed."); + } + } + + private static Dictionary Inventory(string root) + { + var fullRoot = Path.GetFullPath(root); + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + var pending = new Stack(); + pending.Push(fullRoot); + while (pending.Count > 0) + { + foreach (var item in Directory.EnumerateFileSystemEntries(pending.Pop())) + { + var attributes = File.GetAttributes(item); + if ((attributes & FileAttributes.ReparsePoint) != 0) + throw new InvalidDataException("Installed version contains a reparse point."); + if ((attributes & FileAttributes.Directory) != 0) + { + pending.Push(item); + continue; + } + var relative = Path.GetRelativePath(fullRoot, item).Replace('\\', '/'); + if (relative.Equals(FileName, StringComparison.OrdinalIgnoreCase)) continue; + if (!result.TryAdd(relative, item)) + throw new InvalidDataException("Installed version contains duplicate Windows paths."); + } + } + return result; + } + + private static bool SafePath(string path) => + !string.IsNullOrWhiteSpace(path) && !path.Contains('\\') && !Path.IsPathRooted(path) && + path.Split('/').All(segment => segment.Length > 0 && segment is not "." and not ".."); + private static bool SafeVersion(string value) => + System.Text.RegularExpressions.Regex.IsMatch(value, "^[0-9A-Za-z][0-9A-Za-z._-]{0,63}$"); + private static bool FullSha1(string value) => + System.Text.RegularExpressions.Regex.IsMatch(value, "^[0-9a-f]{40}$"); + private static bool Sha256(string value) => + System.Text.RegularExpressions.Regex.IsMatch(value, "^[0-9a-f]{64}$"); +} diff --git a/launcher/OpenBFME.Launcher/LauncherInstallState.cs b/launcher/OpenBFME.Launcher/LauncherInstallState.cs new file mode 100644 index 0000000..7ac5c6e --- /dev/null +++ b/launcher/OpenBFME.Launcher/LauncherInstallState.cs @@ -0,0 +1,93 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace OpenBFME.Launcher; + +internal sealed record LauncherInstallState( + [property: JsonPropertyName("schema")] string Schema, + [property: JsonPropertyName("currentVersion")] string CurrentVersion, + [property: JsonPropertyName("previousVersion")] string? PreviousVersion, + [property: JsonPropertyName("commit")] string Commit, + [property: JsonPropertyName("previousCommit")] string? PreviousCommit, + [property: JsonPropertyName("packageSha256")] string PackageSha256, + [property: JsonPropertyName("previousPackageSha256")] string? PreviousPackageSha256) +{ + private const string ExpectedSchema = "openbfme.launcher-install-state"; + private const string StateName = "launcher-current.json"; + + internal static LauncherInstallState? Load(string root) + { + var path = Path.Combine(root, StateName); + if (!File.Exists(path)) return null; + var state = JsonSerializer.Deserialize(File.ReadAllBytes(path)) + ?? throw new InvalidDataException("Launcher install state is empty."); + state.Validate(); + return state; + } + + internal static void Select(string root, ReleaseManifest manifest, ReleasePackage package) + { + var old = Load(root); + var same = old?.CurrentVersion == manifest.Version; + SaveAtomic(root, new LauncherInstallState( + ExpectedSchema, + manifest.Version, + same ? old?.PreviousVersion : old?.CurrentVersion, + manifest.Commit, + same ? old?.PreviousCommit : old?.Commit, + package.Sha256, + same ? old?.PreviousPackageSha256 : old?.PackageSha256)); + } + + internal static void Rollback(string root) + { + var current = Load(root) ?? throw new InvalidOperationException("No launcher update is installed."); + if (current.PreviousVersion is null || current.PreviousCommit is null || + current.PreviousPackageSha256 is null) + return; + var priorRoot = Path.Combine(root, "launcher-versions", current.PreviousVersion); + InstalledVersionIdentity.Verify( + priorRoot, current.PreviousVersion, current.PreviousCommit, current.PreviousPackageSha256); + BundleInventory.Verify(priorRoot); + SaveAtomic(root, new LauncherInstallState( + ExpectedSchema, + current.PreviousVersion, + current.CurrentVersion, + current.PreviousCommit, + current.Commit, + current.PreviousPackageSha256, + current.PackageSha256)); + } + + private static void SaveAtomic(string root, LauncherInstallState state) + { + state.Validate(); + Directory.CreateDirectory(root); + var path = Path.Combine(root, StateName); + var temporary = path + $".{Environment.ProcessId}.tmp"; + File.WriteAllBytes(temporary, JsonSerializer.SerializeToUtf8Bytes( + state, new JsonSerializerOptions { WriteIndented = true })); + if (File.Exists(path)) File.Replace(temporary, path, null); + else File.Move(temporary, path); + } + + private void Validate() + { + var previousAllNull = PreviousVersion is null && PreviousCommit is null && + PreviousPackageSha256 is null; + var previousAllPresent = PreviousVersion is not null && PreviousCommit is not null && + PreviousPackageSha256 is not null; + if (Schema != ExpectedSchema || !SafeVersion(CurrentVersion) || !FullSha1(Commit) || + !Sha256(PackageSha256) || !(previousAllNull || previousAllPresent) || + previousAllPresent && (!SafeVersion(PreviousVersion!) || !FullSha1(PreviousCommit!) || + !Sha256(PreviousPackageSha256!) || PreviousVersion == CurrentVersion)) + throw new InvalidDataException("Launcher install state is invalid."); + } + + private static bool SafeVersion(string value) => + System.Text.RegularExpressions.Regex.IsMatch(value, "^[0-9A-Za-z][0-9A-Za-z._-]{0,63}$"); + private static bool FullSha1(string value) => + System.Text.RegularExpressions.Regex.IsMatch(value, "^[0-9a-f]{40}$"); + private static bool Sha256(string value) => + System.Text.RegularExpressions.Regex.IsMatch(value, "^[0-9a-f]{64}$"); +} diff --git a/launcher/OpenBFME.Launcher/LauncherOptions.cs b/launcher/OpenBFME.Launcher/LauncherOptions.cs new file mode 100644 index 0000000..3a010ba --- /dev/null +++ b/launcher/OpenBFME.Launcher/LauncherOptions.cs @@ -0,0 +1,98 @@ +namespace OpenBFME.Launcher; + +public sealed record LauncherOptions( + string Channel, + Uri? ManifestUri, + string InstallRoot, + bool NoUpdate, + bool VerifyOnly, + bool Headless, + string? ImportGame, + string? RetailPath) +{ + public static LauncherOptions Parse(IEnumerable arguments) + { + var args = arguments.ToArray(); + string Value(string flag, string fallback) + { + var index = Array.IndexOf(args, flag); + if (index < 0) return fallback; + if (index + 1 >= args.Length || args[index + 1].StartsWith("--", StringComparison.Ordinal)) + throw new ArgumentException($"{flag} requires a value."); + return args[index + 1]; + } + + bool Has(string flag) => Array.IndexOf(args, flag) >= 0; + var channel = Value("--channel", "stable").Trim().ToLowerInvariant(); + if (channel is not ("stable" or "playtest" or "nightly")) + throw new ArgumentException("--channel must be stable, playtest, or nightly."); + + var manifestText = Value("--manifest-url", + channel == "stable" + ? "https://github.com/Ancalgonn/open-bfme-engine/releases/latest/download/release-manifest.json" + : ""); + Uri? manifestUri = null; + if (manifestText.Length > 0) + { + if (!Uri.TryCreate(manifestText, UriKind.Absolute, out manifestUri)) + throw new ArgumentException("--manifest-url must be an absolute URL."); + ReleaseUriPolicy.Validate(manifestUri); + } + + var bfme2 = Has("--import-bfme2"); + var rotwk = Has("--import-rotwk"); + if (bfme2 && rotwk) throw new ArgumentException("Select only one import game per run."); + var game = bfme2 ? "bfme2" : rotwk ? "rotwk" : null; + var retail = game == "bfme2" ? Value("--bfme2-path", "") : + game == "rotwk" ? Value("--rotwk-path", "") : ""; + if (game is not null && retail.Length == 0) + throw new ArgumentException($"--import-{game} requires its retail path flag."); + + var root = Path.GetFullPath(Value( + "--install-root", + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "OpenBFME"))); + return new LauncherOptions(channel, manifestUri, root, Has("--no-update"), + Has("--verify-only"), Has("--headless"), game, + retail.Length == 0 ? null : Path.GetFullPath(retail)); + } +} + +public static class ReleaseUriPolicy +{ + private const string RepositoryReleasePrefix = + "/Ancalgonn/open-bfme-engine/releases/"; + private static readonly HashSet RedirectHosts = new(StringComparer.OrdinalIgnoreCase) + { + "objects.githubusercontent.com", + "github-releases.githubusercontent.com", + "release-assets.githubusercontent.com" + }; + + public static void Validate(Uri uri) + { + ValidateCommon(uri); + if (!uri.DnsSafeHost.Equals("github.com", StringComparison.OrdinalIgnoreCase) || + !uri.AbsolutePath.StartsWith(RepositoryReleasePrefix, StringComparison.OrdinalIgnoreCase) || + !string.IsNullOrEmpty(uri.Query)) + throw new InvalidOperationException("Release URL does not belong to the approved repository."); + } + + public static void ValidateResponse(Uri uri) + { + ValidateCommon(uri); + if (uri.DnsSafeHost.Equals("github.com", StringComparison.OrdinalIgnoreCase)) + { + Validate(uri); + return; + } + if (!RedirectHosts.Contains(uri.DnsSafeHost)) + throw new InvalidOperationException("Release redirect host is not approved."); + } + + private static void ValidateCommon(Uri uri) + { + if (uri.Scheme != Uri.UriSchemeHttps || uri.UserInfo.Length != 0 || + !string.IsNullOrEmpty(uri.Fragment)) + throw new InvalidOperationException("Release URLs must use credential-free HTTPS."); + } +} diff --git a/launcher/OpenBFME.Launcher/LauncherSelfUpdate.cs b/launcher/OpenBFME.Launcher/LauncherSelfUpdate.cs new file mode 100644 index 0000000..951c3b3 --- /dev/null +++ b/launcher/OpenBFME.Launcher/LauncherSelfUpdate.cs @@ -0,0 +1,30 @@ +using System.Diagnostics; + +namespace OpenBFME.Launcher; + +internal static class LauncherSelfUpdate +{ + internal static bool RelaunchSelected(LauncherOptions options, IReadOnlyList arguments) + { + var state = LauncherInstallState.Load(options.InstallRoot); + if (state is null) return false; + var selectedRoot = Path.Combine( + options.InstallRoot, "launcher-versions", state.CurrentVersion); + var selected = Path.Combine(selectedRoot, "OpenBFME.Launcher.exe"); + var current = Path.GetFullPath( + Environment.ProcessPath ?? throw new InvalidOperationException("Launcher process path is unavailable.")); + if (Path.GetFullPath(selected).Equals(current, StringComparison.OrdinalIgnoreCase)) + return false; + InstalledVersionIdentity.Verify( + selectedRoot, state.CurrentVersion, state.Commit, state.PackageSha256); + BundleInventory.Verify(selectedRoot); + var start = new ProcessStartInfo(selected) + { + UseShellExecute = false, + WorkingDirectory = selectedRoot + }; + foreach (var argument in arguments) start.ArgumentList.Add(argument); + Process.Start(start); + return true; + } +} diff --git a/launcher/OpenBFME.Launcher/LauncherService.cs b/launcher/OpenBFME.Launcher/LauncherService.cs new file mode 100644 index 0000000..8cd0060 --- /dev/null +++ b/launcher/OpenBFME.Launcher/LauncherService.cs @@ -0,0 +1,35 @@ +using System.Diagnostics; + +namespace OpenBFME.Launcher; + +public sealed class LauncherService +{ + public LauncherOptions Options { get; } + public ReleaseInstaller Installer { get; } = new(); + public ImporterRunner Importer { get; } = new(); + + public LauncherService(LauncherOptions options) => Options = options; + + public InstallState? Current => InstallState.Load(Options.InstallRoot); + + public string CurrentGamePath() + { + var state = Current ?? throw new InvalidOperationException("OpenBFME is not installed."); + var root = Path.Combine(Options.InstallRoot, "versions", state.CurrentVersion); + ReleaseInstaller.VerifyInstalledVersion(root, state.CurrentVersion, state.Commit); + return Path.Combine(root, "OpenBFME.exe"); + } + + public void LaunchGame() + { + var exe = CurrentGamePath(); + var start = new ProcessStartInfo(exe) + { + UseShellExecute = false, + WorkingDirectory = Path.GetDirectoryName(exe)! + }; + start.Environment["OPENBFME_CONTENT"] = Path.Combine( + Options.InstallRoot, ".private", "content-packs"); + Process.Start(start); + } +} diff --git a/launcher/OpenBFME.Launcher/MainWindow.xaml b/launcher/OpenBFME.Launcher/MainWindow.xaml new file mode 100644 index 0000000..f540221 --- /dev/null +++ b/launcher/OpenBFME.Launcher/MainWindow.xaml @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +