diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index ca9f20d..021c82d 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -88,6 +88,12 @@ jobs: # Read with LEIALOFT_GITHUB_TOKEN — the same secret the Android arm uses # for LeiaInc/CNSDK. Fork PRs have no secret and cannot build this arm. SR_SDK_REPO: LeiaInc/SR-SDK-Windows-Releases-Internal-Public + # NeurD headers for the 2D->3D lift module (docs/lift-neurd.md), fetched + # from the PRIVATE LeiaInc/media_sdk repo into the gitignored + # NeurD-SDK-/include -- never committed. Pin a media_sdk release tag. + # KEEP IN SYNC with scripts/build-windows.bat (check_sr_pins.py asserts). + NEURD_SDK_REF: v0.4.6 + NEURD_SDK_REPO: LeiaInc/media_sdk GH_TOKEN: ${{ github.token }} # SourceForge serves a bot-check page to CI runners, so vcpkg's pthreads # port (vcpkg_from_sourceforge) fails its hash on every mirror. This @@ -103,7 +109,7 @@ jobs: # regression originally). # #734: runtime v2.0.0 (a8f6e74f) ships the XR_EXT_* → XR_DXR_* # extension rename this plug-in's headers now use (ABI unchanged, v4). - RUNTIME_REF: v2.16.9 + RUNTIME_REF: v2.22.0 steps: - name: Checkout uses: actions/checkout@v4 @@ -180,6 +186,21 @@ jobs: if (-not (Test-Path "$env:SR_V2_DIR/$f")) { throw "SR v2 SDK missing $f" } } + # SOFT: without the headers the lift module compiles out and the plug-in + # is otherwise identical, so a missing token or an unreachable repo must + # not fail the build -- it is surfaced as a warning annotation instead. + - name: Fetch NeurD headers + if: needs.DetectChanges.outputs.docs_only != 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.LEIALOFT_GITHUB_TOKEN }} + run: | + ./scripts/fetch-neurd-headers.ps1 -Repo $env:NEURD_SDK_REPO -Ref $env:NEURD_SDK_REF -Dest "NeurD-SDK-$env:NEURD_SDK_REF" + if ($LASTEXITCODE -ne 0) { + Write-Host "::warning::NeurD headers not fetched ($env:NEURD_SDK_REPO@$env:NEURD_SDK_REF) -- this build ships WITHOUT 2D->3D lift." + exit 1 + } + # The plug-in CMake adds the runtime as a subdirectory, so we need # all the runtime's own build prereqs (vcpkg, Vulkan SDK, OpenXR # loader). Same shape as the runtime's build-windows.yml. @@ -282,6 +303,8 @@ jobs: env: LEIASR_SDKROOT: ${{ github.workspace }}/LeiaSR-SDK-${{ env.SR_TAG }}-win64 LEIASR_V2_SDKROOT: ${{ github.workspace }}/${{ env.SR_V2_DIR }} + # Read by src/drv_leia/CMakeLists.txt; absent dir => lift compiles out. + NEURD_SDK_ROOT: ${{ github.workspace }}/NeurD-SDK-${{ env.NEURD_SDK_REF }} run: | cmake -S . -B build -G "Ninja Multi-Config" ` -DDXR_RUNTIME_SOURCE_DIR="${{ github.workspace }}/displayxr-runtime" ` diff --git a/.gitignore b/.gitignore index e55eba9..a813734 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ __MACOSX/ # Fetched SR SDK (not committed; downloaded by CI / scripts/build-windows.bat) LeiaSR-SDK-*/ simulatedreality-SDK-*/ +# Fetched NeurD headers (Leia-private; scripts/fetch-neurd-headers.ps1) +NeurD-SDK-*/ # Installer artifacts (built into _package/) DisplayXRLeiaSRSetup-*.exe diff --git a/CLAUDE.md b/CLAUDE.md index 669e5d9..09aa1fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,6 +98,7 @@ ADR-020 spec: [`displayxr-runtime/docs/adr/ADR-020-plugin-abi-policy.md`](https: | `src/drv_leia/leia_display_processor*.{cpp,h}` | Per-API display processors (D3D11, D3D12, GL). Implement `xrt_display_processor_` vtables; the runtime's compositor invokes them per-frame. | | `src/drv_leia/leia_sr_*.{cpp,h}` | SR SDK weaver wrappers. The SDK throws `std::runtime_error` as routine internal control flow (~11/frame in some paths) — every call wrapped in try/catch. | | `src/drv_leia/leia_bg_capture_win.{cpp,h}` | WGC background capture for compose-under transparency (Leia transparency model). | +| `src/drv_leia/leia_lift_neurd.{cpp,h}` | 2D→3D lift slots on the D3D11 DP, backed by a dynamically loaded NeurD.dll (absent ⇒ caps unavailable, nothing loaded). Compiles only with the private NeurD headers (fetched at build time, below) AND runtime headers with `XRT_DP_D3D11_HAS_LIFT`; otherwise the slots stay NULL. See [`docs/lift-neurd.md`](docs/lift-neurd.md). | | `src/drv_leia/leia_edid_probe.c` | EDID-based hardware detection — answers "is a Leia display attached?" before the SR SDK initializes. | | `src/drv_leia_linux/leia_sr_linux.h` | **Linux weaver-backend seam** — interface shaped 1:1 by `docs/leia-linux-sdk-contract.md` (every declaration cites its R-* requirement). Track B implements it against the real SDK. | | `src/drv_leia_linux/leia_sr_stub.c` | Track A stub backend: canned panel info + passthrough SBS blit, `TODO(Track B)` at every body. | @@ -134,6 +135,15 @@ run by `.github/workflows/lint.yml` on every PR, asserts all five pins declared exactly once and identically in both files, and that `SR_V2_TAG`/`SR_V2_DIR` name the same SR v2 build. +**NeurD headers (2D→3D lift) follow the same rule.** They are Leia-private +and come from the private **`LeiaInc/media_sdk`** repo at `NEURD_SDK_REF` +(pinned next to the SR pins in both files, also checked by +`check_sr_pins.py`), fetched by `scripts/fetch-neurd-headers.ps1` into the +gitignored `NeurD-SDK-/`. **Never commit or mirror them.** Your `gh` +account (and CI's `LEIALOFT_GITHUB_TOKEN`) needs read access to +`media_sdk`; without it the fetch fails SOFT and the plug-in builds without +lift. Details: [`docs/lift-neurd.md`](docs/lift-neurd.md). + ### Linux ```bash ./scripts/build-linux.sh # Track A stub: .so + displayxr-cli, stage manifest, selftest diff --git a/CMakeLists.txt b/CMakeLists.txt index ea8425c..a5d88d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,9 +162,13 @@ endif() # This plug-in MUST be built against headers that ship the matching major, # or the runtime will log "ABI major mismatch — plugin_api=N, runtime # expects M ... skipping (ADR-020 rule 3)" and fall back to sim_display. +# v2.22.0 (from v2.16.9): the first runtime that ships the D3D11 DP lift slots +# (XRT_DP_D3D11_HAS_LIFT, runtime#1781) and the lift-only factory +# (XRT_PLUGIN_IFACE_HAS_D3D11_LIFT_FACTORY). Appended and struct_size-gated, +# plug-in ABI still v5; an older pin compiles the NeurD lift module OUT. # The rule-5 CI self-check below asserts this value equals the workflow's # runtime checkout ref so the two can't drift. -set(DXR_RUNTIME_GIT_TAG "v2.16.9" CACHE STRING +set(DXR_RUNTIME_GIT_TAG "v2.22.0" CACHE STRING "Runtime git ref to fetch. Must equal the workflow's runtime checkout ref (rule-5 CI check enforces).") # Linux carries its own pin so it can move independently of the Windows # installer's derived MIN_RUNTIME_VERSION (same reason as _ANDROID below). diff --git a/docs/README.md b/docs/README.md index 38b3c0b..8cc3d13 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,7 @@ Leia is the first 3D-display vendor integrated into DisplayXR. This directory do | Display processor (per API) | `leia_display_processor.{cpp,h}` (base), `leia_display_processor_d3d11.{cpp,h}`, `leia_display_processor_d3d12.{cpp,h}`, `leia_display_processor_gl.{cpp,h}` | | Weaver (per API) | `leia_sr.{cpp,h}` (base + eye tracking), `leia_sr_d3d11.{cpp,h}`, `leia_sr_d3d12.{cpp,h}`, `leia_sr_gl.{cpp,h}` | | Background capture (transparency) | `leia_bg_capture_win.{cpp,h}` | +| 2D→3D lift (NeurD, D3D11) | `leia_lift_neurd.{cpp,h}` (+ private NeurD headers fetched at build time) | | Shaders | `shaders/` | ## Docs in this directory @@ -21,6 +22,7 @@ Leia is the first 3D-display vendor integrated into DisplayXR. This directory do - **[Weaver internals](weaver.md)** — DX11 / DX12 / OpenGL / Vulkan weaver creation, inputs, weave() flow, DPI handling, phase math. - **[Transparency model](transparency.md)** — current primary path: WGC compose-under-bg on D3D11 / D3D12 / Vulkan. Replaces the older chroma-key approach for those APIs. - **[Chroma-key overlay (legacy / OpenGL fallback)](chroma-key-overlay.md)** — fallback path; still the only transparency path on the Leia OpenGL DP. +- **[2D→3D lift via NeurD](lift-neurd.md)** — the D3D11 DP's `lift_*` slots: dynamic NeurD.dll load, one-per-process activation (licence network failure = *activating*, retried), the shared-texture device bridge to NeurD's own D3D11 device, tracked-eye → NeurD viewpoint mapping, the `DXR_LEIA_LIFT*` knobs, and how to verify with `displayxr-cli lift caps|probe`. - **[Late latching](late-latching.md)** — which backends implement it (D3D11/GL automatic, VK needs our submit hook, D3D12 is a stub), what `srWeaverIsLateLatchingEnabled` hides, the once-per-frame / deferred-context contract from the shipped header, and the dot test — including why `pattern = 450` makes it lie. - **[SR panel readiness and late geometry](sr-readiness.md)** — the boot race (SR "default display" until the FPC serial link answers; never while the panel sleeps), the two readiness signals that actually work (`Global\sharedDeviceSerialMemory` count byte, a *fresh* valid display handle), the ONE shared startup budget (`DXR_LEIA_SR_READY_TIMEOUT_S`, default 20 s), the 1 Hz late-identification watcher that re-derives geometry and updates the live head device in place, and the `get_display_info` contract with the runtime (#266). - **[Window phase snapping](window-phase-snapping.md)** — the SR weaver's automatic `WndProc` subclassing for lenticular phase alignment during window drag (Windows), and the resolved `WndProcDispatcher` race. diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md new file mode 100644 index 0000000..6eb80f1 --- /dev/null +++ b/docs/lift-neurd.md @@ -0,0 +1,364 @@ +# 2D→3D lift via NeurD (Windows, D3D11) + +The Leia D3D11 display processor implements the runtime's **lift** slots — convert a +single 2D RGBA frame into depth, a side-by-side stereo pair, or an N-view grid — by +driving Leia's **NeurD** library. Source: `src/drv_leia/leia_lift_neurd.{h,cpp}` +(module) and the slot glue at the bottom of `src/drv_leia/leia_display_processor_d3d11.cpp`, +built against the real NeurD headers fetched at build time (see *Building*). + +The runtime owns the lift *policy* — which content is lifted, the async worker thread, +the result mailbox/ring, and how the output reaches the compositor. The plug-in owns +only "turn this texture into that texture, now". + +## Contract (runtime side) + +Slots appended to `struct xrt_display_processor_d3d11` (runtime `xrt_dp_lift.h`, +ADR-042), announced by `XRT_DP_D3D11_HAS_LIFT` (runtime branch `feat/lift-ext`). The +plug-in fills the four texture-mode slots; the fifth, `lift_convert_blob` +(GAUSSIANS, photo → splats), is left NULL — NeurD has no splat path — so `modes` never +carries bit 8. + +| Slot | Plug-in behaviour | +|---|---| +| `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS (1\|2) once NeurD is present, else 0. **Never NVIEW**: not available on NeurD ≤ 0.4.6 (public output types fix the tile grid); SBS with explicit viewpoints only. `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 8, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | +| `lift_stream_create` | Non-blocking. Succeeds while NeurD is still activating — the NeurD stream is created lazily on the first convert. | +| `lift_stream_destroy` | Releases the stream's NeurD stream and bridge resources. | +| `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device — `R8G8B8A8_UNORM` for SBS, `R8_UNORM` for DEPTH (polarity flipped at the bridge, in the R8 unpack, from NeurD's near = high disparity to the spec's RELATIVE larger = farther) — owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | + +Every versioned struct is read only as far as its `struct_size` covers. +`xrt_dp_lift_params.focal_px` (appended) is ignored — it only matters to a +photo → Gaussians module, and `lift_convert_blob` stays NULL. + +### The lift-only DP (`create_dp_d3d11_lift`) + +The plug-in fills the optional `xrt_plugin_iface::create_dp_d3d11_lift` factory +(runtime header macro `XRT_PLUGIN_IFACE_HAS_D3D11_LIFT_FACTORY`). The runtime creates +exactly one lift DP per process on a dedicated device, calls only the `lift_*` slots on +it from its lift thread, and always passes explicit `viewpoints_xyz`. So this DP has +**no SR weaver, no window, no tracker, no lens control** — just the NeurD handle; every +non-lift slot is NULL. This is what keeps a lift session from perturbing the lens. +The weaving DP (`create_dp_d3d11`) still carries the lift slots too (they load nothing +unless called), which is what a runtime without the lift factory falls back to. + +Output layouts (runtime contract): DEPTH = one channel at the inference resolution +(NeurD writes the normalised disparity replicated into RGB; the unpack keeps R); +SBS = 2 views side by side. (NVIEW — `view_count` views in one row — is not offered +by this module; see *N-view* below.) Every convert checks NeurD's actual layout +(output aspect ÷ input aspect = tile count) against the mode and drops a frame that +doesn't match, so a result can never be mislabelled. Tile size = the *inference* resolution (after autoscaling), not the +input size — use `out_w/out_h`. One row is why `max_views` is 8: 8 × 2560 (1440p) is +the widest that fits D3D11's 16384-texel limit; a wider request fails with a WARN. + +## Process model + +- **Dynamic load only.** `NeurD.dll` is found in NeurD's own loader order: `PATH` → + `NEURD_PATH` (full path to the DLL) → `HKLM\SOFTWARE\LeiaInc\NeurD` (default value = + install dir), loaded with `LOAD_WITH_ALTERED_SEARCH_PATH`. No import lib: the module + resolves only the DLL's `NeurD_load` export and then calls through the returned + function table using the **real** `NeurD.h`'s header-inline wrappers and PFN types, + which version-gate every entry (an older runtime's table is shorter) — so any NeurD + signature drift is a compile error, not a runtime mismatch. +- **Absent NeurD → nothing changes.** A presence probe (no `LoadLibrary`) runs on the + first caps/stream call; if the DLL is not found the process state becomes + *absent*, caps report `modes=0 / state=0`, and the plug-in behaves exactly as before. +- **One NeurD instance per process** (NeurD's rule). DP handles ref-count it. It is + loaded and initialised on a **detached background thread**, never on a caller's + thread and never at DP create (the DP factory runs on the service critical path). +- **Never de-initialised.** DPs are recreated on focus changes; re-init would re-run + licensing and model load. When the last DP handle goes the plug-in only calls + `NeurD_shrink_memory_pool`. + +### Licensing / activation + +`NeurD_init_with_options(multithreaded = TRUE)` runs NeurD's LexActivator licence +check. First activation on a machine needs the network (and WMI). Outcomes: + +| NeurD result | Plug-in state | Caps `state` | +|---|---|---| +| `SUCCESS` | ready | 2 | +| `LICENSE_NETWORK_ERROR` / `NETWORK_ERROR` | retry-wait; re-attempted on the next caps/stream/convert call ≥ 15 s later | 1 (activating) | +| `INVALID_LICENSE`, model/filesystem errors, anything else | failed for the process | 0 | + +Each transition logs one WARN (`Leia lift: ...`). + +### Backend + +`NeurD_set_backend` is called immediately before init; NeurD keeps a forced backend for +the process, so the **first** DP to activate NeurD decides it. On NeurD 0.4.3+ the D3D11 +path needs NeurD's D3D11 device (`NeurD_get_dx_device`), which only the **DirectML** +backend provides — CUDA and OpenVINO return none, and lift then reports unavailable with +a WARN naming the fix. Hence the default is `directml`, not NeurD's own `auto`. NeurD +0.3.x is the other way round — see below. + +### NeurD 0.3.x + +Supported from **0.3.11** (the first release with the stream API: `create_stream`, +`convert_stream_dx`, `set_prop_1i/1f`). The function table is append-only, so a plug-in +built against the 0.4.x header loads a 0.3.x `NeurD.dll` unchanged, and every newer entry +is version-gated (`LEIA_NEURD_HAS`). + +- **`NeurD_get_dx_device` is optional** (added in 0.4.3). Without it the plug-in creates + its own D3D11 device on the **default adapter** (as NeurD's own example does), builds + the bridge on it, and releases it itself. 0.3.x's DX convert takes the device from the + input buffer (`GetDevice`) and returns its output as an `R8G8B8A8_UNORM` texture on that + same device, so the bridge is otherwise unchanged. One WARN names the case and the LUID. +- **CUDA backend required for the DX path on 0.3.x.** Its DX convert is implemented on + CUDA↔D3D11 interop; the DirectML build's DX path is not verified. Set + backend `cuda` — `DXR_LEIA_LIFT_BACKEND=cuda` in the **service's** environment, or + better the `Backend` registry value (see *Knobs*), which survives service respawns. The default adapter must be the NVIDIA GPU for + the interop to bind — on a hybrid box, pin the service to the dGPU. +- **SBS and DEPTH only** (as on every NeurD today — see *N-view*); without interactive + convert (0.4.5+) tracked-eye viewpoints are ignored (SBS uses NeurD's default pattern). + Because 0.3.x hands back a texture, DEPTH arrives as `R8G8B8A8_UNORM` (copied as-is), + not `R8_UNORM`; `lift_convert`'s `out_format` reports which. That texture path is **not** + polarity-flipped: DEPTH stays NeurD's near = high, contrary to RELATIVE (known gap). +- **Default models only** — `init_with_options` (model selection) is newer; `NeurD_init` + is used instead. + +## Threading + +- Every entry point may be called from any thread. `lift_convert` blocks for tens of + ms: the runtime calls it only from its lift thread. +- NeurD properties (output type, tile grid, gain, convergence, inpaint, autoscale) are + **process-global** inside NeurD, so one process-wide mutex makes *set properties → + convert → stage output* atomic across streams. Properties are re-sent only when they + change. +- The caller's immediate context must be `ID3D11Multithread`-protected (NeurD's and our + own copies run on it from the lift thread while the compositor renders). The plug-in + enables protection if it is off — the same mechanism the async weaver already relies + on (`enable_context_multithread_protection`, `leia_sr_d3d11.cpp`). NeurD's own + immediate context is protected too. + +## Device bridge + +NeurD runs on **its own** D3D11 device, possibly on a different adapter than the +runtime's service device on hybrid boxes. Its DX input must be a flat RGBA8 +`ID3D11Buffer` (stride `w*4`) with `D3D11_RESOURCE_MISC_SHARED`, and it hands the +output back on the input buffer's device. Buffers do not share reliably across devices; +textures do. So (the LeiaMeet `DxStereoConverter` pattern): + +``` +caller device NeurD device +------------- ------------ +input tex ──CopySubresourceRegion──▶ in_bridge (shared tex, created on NeurD's device) + event-query drain (CPU) + in_bridge SRV ──pack CS──▶ raw RGBA8 buffer ; drain + NeurD_convert_stream_dx[_interactive] (blocking) + NeurD output ──unpack CS / copy──▶ out_bridge ; drain +out_bridge (opened) ◀───────────────── returned to the runtime +``` + +Each hop is CPU-drained with an event query because NeurD's DirectML work runs on its +own queue, unordered against any D3D11 context. Bridges are per stream, sized lazily, +and rebuilt on a size/format/device change. The adapter LUIDs of both devices are +logged once per stream (`same-GPU copy` vs `cross-adapter copy`); the bridge is used +either way. Accepted inputs: single-sampled `R8G8B8A8_*` or `B8G8R8A8_*` textures +(the pack shader reads through a typed view, so BGRA is swizzled for free and sRGB +bytes pass through encoded — what NeurD expects). + +## Tracked eyes → NeurD viewpoints + +NeurD's `*_interactive` converts take one `(x, y, z)` triplet per output view in +**dimensionless model units**: `x` horizontal disparity gain, `y` vertical offset, `z` +depth offset. Its default stereo pattern is `x = −0.5 / +0.5`, i.e. one unit of `x` is +one nominal eye baseline. The plug-in maps display-space positions in **metres** (the +same eyes `get_predicted_eye_positions` reports) with: + +``` +x_n = clamp(G · x_m / 0.063, ±3) +y_n = clamp(G · y_m / 0.063, ±3) +z_n = 0 +G = DXR_LEIA_LIFT_VIEW_GAIN (default 1.0) +``` + +A centred viewer at 63 mm IPD therefore reproduces NeurD's default pattern exactly +(G = 1), and head motion becomes look-around parallax. `G = 0` pins every view to the +centre; `G > 1` exaggerates. Head `z` is deliberately not mapped: NeurD does not specify +how its `z` relates to viewing distance, so mapping it would be a guess. + +Viewpoint source, in order: + +1. **Explicit viewpoints** from the runtime (always sent to the lift-only DP) (`viewpoints_xyz`, metres, display space): + `view_count` triplets are used 1:1; exactly two are treated as an eye pair. +2. **Tracked eyes** from the SR eye path, when tracking is live (inter-eye distance + > 1 mm — the same tracking-loss test as the eye slot). +3. **Untracked**: SBS uses NeurD's own default pattern (non-interactive convert); NVIEW + uses `x = i − (N−1)/2`. + +For N views from an eye pair, views are spaced one eye baseline apart and centred on +the eye midpoint. If NeurD predates the interactive API (< 0.4.5), SBS falls back to the +default pattern. (The N-view paths are unreachable today: NVIEW streams are refused.) + +## N-view + +**Not available on NeurD ≤ 0.4.6.** NeurD's `Config::adjust()` runs on every convert +(including the interactive one) and re-derives the tile grid from the output type — +SBS → 2×1, TB → 1×2, DEPTH → 1×1 — and the public `NEURD_PROP_OUTPUT_TYPE` accepts only +those, so `OUTPUT_TILES_W/H` are overridden. Panel-verified: a 4-view request came back +as a plain 2-tile SBS. Caps never advertise NVIEW, and `lift_stream_create(NVIEW)` fails +with a WARN. What works: SBS with explicit viewpoints (two, from the runtime or tracked +eyes). + +### Asks for Leia media_sdk + +- A tiled / multiview output type usable with `convert_stream_dx_interactive` + (N viewpoints → N tiles in one convert). +- A public release of the 0.4.4 *internal-interactive* build (interactive entries in a + build whose reported version matches its table). +- A working 0.4.6 package. + +## Parameters + +| `xrt_dp_lift_params` | NeurD | +|---|---| +| `convergence < 0` | `AUTO_CONVERGENCE = TRUE` | +| `convergence` in [0, 1] | `AUTO_CONVERGENCE = FALSE`, `CONVERGENCE = clamp(K · (c − 0.5), ±0.2)`, K = `DXR_LEIA_LIFT_CONV_GAIN` | +| `strength` | `GAIN_MULTIPLIER` (1 = NeurD's calibrated budget, 0 = flat), clamped [0, 10]; negative → 1.0 | +| `inpaint` | NeurD always fills disocclusions: 0 → `V1_STRETCH` (cheapest), non-zero → `V1_BLUR` (the DX video path supports V1 only) | +| `view_count` | NVIEW row width — unused (NVIEW not offered on NeurD) | + +**Convergence calibration.** The runtime's `convergence` is the relative depth placed +at the display plane, normalised to [0, 1] over the frame's depth range (0 = nearest +content on the glass, 1 = farthest, 0.5 = middle). NeurD's `CONVERGENCE` is its own +disparity offset in [−0.2, 0.2]. The plug-in maps linearly about mid-range with gain +K (default 0.4, which spans NeurD's whole range). This is **uncalibrated**: on a panel, +submit c = 0 and check that the nearest content sits on the glass; if the far content +does instead, the sign is reversed — set `DXR_LEIA_LIFT_CONV_GAIN=-0.4`; then tune |K| +until c = 0 and c = 1 put the extremes on the glass. Auto-convergence (the default, +c < 0) is unaffected. + +`xrt_dp_lift_stream_info.input_scale` in (0, 1] (1 = native) picks the smallest NeurD +autoscaling bucket (720p / 1080p / 1440p / none) that covers `input_scale × input +height`; outside that range the 720p default applies, and an explicitly set +`DXR_LEIA_LIFT_SCALE` overrides every stream. `content_hint` is recorded but +advisory: NeurD's DX stream path always runs its video model (photo mode is an +init-time, process-wide property). + +Depth output is NeurD's **relative** disparity, min-max normalised per frame at the +inference resolution. + +## Knobs + +Read once per DP at create (`leia_lift_neurd_create`). Each knob comes from the +**environment**, else the **registry** (`HKLM\SOFTWARE\DisplayXR\Leia\Lift`, REG_SZ, +64-bit view, same grammar as the env var), else its default — env > registry > default. +At activation one WARN lists every knob's effective value and where it came from: + +``` +Leia lift: knobs backend=directml(reg) interactive_min=0.4.4(reg) scale=per-stream(default) view_gain=1.00(default) conv_gain=0.40(default) [env > HKLM\SOFTWARE\DisplayXR\Leia\Lift > default] +``` + +**Why the registry.** Under the service the knobs are read from `displayxr-service.exe`'s +environment, not the client's. Any respawn — tray relaunch, the HKLM `Run` key at logon, +a crash restart — starts the service with the logon environment, and env-only knobs +vanish silently (the only tell was "interactive viewpoints UNAVAILABLE"). A registry +value survives every respawn. Env still wins when set, so `.bat`-driven runs behave as +before; with neither set, nothing changes. + +| Env | Registry value | Default | Effect | +|---|---|---|---| +| `DXR_LEIA_LIFT` | — (env only) | on | `0` / `off` → caps `modes=0, state=0`; NeurD is never probed or loaded. | +| `DXR_LEIA_LIFT_BACKEND` | `Backend` | `directml` | `auto` \| `directml` \| `cuda` \| `openvino`. First activation in the process wins (NeurD's forced backend is sticky). On NeurD 0.4.3+ only DirectML yields a D3D11 device; on 0.3.x use `cuda` (see *NeurD 0.3.x*). | +| `DXR_LEIA_LIFT_SCALE` | `Scale` | unset (→ stream `input_scale`, else 720p) | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. When set it overrides every stream's `input_scale`. NeurD's own default is 1440p; 720p is the fallback for latency. | +| `DXR_LEIA_LIFT_VIEW_GAIN` | `ViewGain` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | +| `DXR_LEIA_LIFT_CONV_GAIN` | `ConvGain` | `0.4` | `K` in the convergence map above, [−2, 2]; negative flips the sign. Calibration knob. | +| `DXR_LEIA_LIFT_INTERACTIVE_MIN` | `InteractiveMin` | unset (→ header, 0.4.5) | **Demo-only.** A NeurD version, e.g. `0.4.4` (clamped to ≥ 0.4.4), from which `convert_stream_dx_interactive` is trusted. For the 0.4.4 *internal-interactive* dev package, which reports 0.4.4 but carries the interactive entries. The version is the only discriminator (only `NeurD_load` is exported, and a stock 0.4.4 table is too short to probe), so on a **stock 0.4.4 this crashes** — never set it elsewhere. One extra WARN when in effect. When it admits a NeurD older than 0.4.5, the plug-in calls the table slot directly (the header's inline wrapper re-checks 0.4.5). | + +The two demo knobs, set once on the box (elevated prompt), then restart the service: + +```bat +reg add "HKLM\SOFTWARE\DisplayXR\Leia\Lift" /v Backend /t REG_SZ /d directml /f /reg:64 +reg add "HKLM\SOFTWARE\DisplayXR\Leia\Lift" /v InteractiveMin /t REG_SZ /d 0.4.4 /f /reg:64 +``` + +`scripts\set-lift-knobs.bat` does exactly this (`--clear` deletes the key). + +## Building + +Lift compiles in only when BOTH of these hold; otherwise the four slots stay NULL, the +runtime reports lift unavailable, and the rest of the plug-in is unchanged. Neither +condition ever fails the plug-in build. + +### 1. The NeurD headers (private) + +`NeurD.h` and its closure (`_NeurD_detail.h`, `_NeurD_table.h`, and a generated +`NeurD_version.h`) are **Leia-private** and are never committed here. They are fetched +from the private **`LeiaInc/media_sdk`** repo at a pinned ref by +`scripts/fetch-neurd-headers.ps1` into the gitignored `NeurD-SDK-/include`, +exactly like the SR SDK: + +| | | +|---|---| +| Pins | `NEURD_SDK_REF` + `NEURD_SDK_REPO` in `scripts/build-windows.bat` **and** `.github/workflows/build-windows.yml` (`jobs.Build.env`), kept equal by `scripts/check_sr_pins.py` (lint.yml). | +| Current pin | `v0.4.6` — the newest media_sdk release tag; its headers are byte-identical to `dev@25a713d93` (the tip this module was written against). Move it to a newer **release tag** when the module needs a newer NeurD API. | +| Auth | `gh` with read access to `LeiaInc/media_sdk` (`gh auth login`, or `GH_TOKEN`). CI uses `secrets.LEIALOFT_GITHUB_TOKEN`, the SR SDK token — that token must be granted read on `media_sdk`. | +| Failure | Soft. The .bat prints a WARN and builds without lift; CI emits a `::warning::` annotation (`continue-on-error`) and ships without lift. CMake prints `NeurD headers NOT found ... lift compiled OUT`. | +| Override | `set NEURD_SDK_ROOT=` (e.g. a local media_sdk drop) skips the fetch. | + +`NeurD_version.h` is generated the way media_sdk's CMake does it, from +`sdk/NeurD_version.h.in` and the top-level `project(mediasdk VERSION x.y.z)` at the same +ref; it is the table version the module requests from `NeurD_load`. + +### 2. Runtime headers with the lift slots + +The slots compile only when the runtime headers define `XRT_DP_D3D11_HAS_LIFT`, which +first ships in runtime `v2.22.0` — the Windows pin `DXR_RUNTIME_GIT_TAG`. Built against +older headers the plug-in ships with the lift slots compiled out (it logs +`lift slots NOT COMPILED`), and the runtime sees them absent via `struct_size`. + +To build against an unreleased runtime, use a local runtime checkout — nothing +is committed: + +```bat +:: gh must be able to read LeiaInc/media_sdk (and the SR SDK repo, as always) +gh auth status +set DXR_RUNTIME_SOURCE_DIR=C:\dev\displayxr-runtime.wt-lift +scripts\build-windows.bat build +``` + +Confirm the configure output says `NeurD headers found ... enabling the D3D11 2D->3D lift +slots`, and at runtime the service log says `lift slots WIRED`. Set +`DXR_RUNTIME_SOURCE_DIR` explicitly: the script otherwise auto-detects a sibling +`displayxr-runtime` checkout, which may not carry the lift headers. + +or `cmake -DDXR_RUNTIME_GIT_TAG=feat/lift-ext ...` for a fetched build. **Do not commit +a branch or SHA pin**: `installer/CMakeLists.txt` derives `MIN_RUNTIME_VERSION` from +`DXR_RUNTIME_GIT_TAG`, and the orchestrator/meta-installer assume a `vX.Y.Z` tag. Once +the runtime ships the lift contract in a tag, move `DXR_RUNTIME_GIT_TAG` (and the +workflow's runtime ref, which the rule-5 check keeps equal) to that tag in the plug-in +release that should carry lift. Because the slots are appended (ADR-020), that re-pin +raises the installer floor to the lift tag — the price of shipping the capability. + +## Verifying + +On a Windows box with NeurD installed (`HKLM\SOFTWARE\LeiaInc\NeurD` present) and the +lift-enabled runtime + this plug-in registered: + +1. `displayxr-cli lift caps` — first call reports `state=activating` (the background + worker is loading NeurD); repeat until `state=ready`, `modes=0x7`, + `backend=neurd-directml`. The runtime log shows `Leia lift: loaded NeurD x.y.z from + ...` and `Leia lift: NeurD READY`. +2. `displayxr-cli lift probe` — runs a stream create + convert + destroy. Check the log + for the adapter-LUID line and no `Leia lift:` WARN after it; the probe's reported + latency should sit near 22 ms (DirectML) at 720p. +3. Absent-NeurD check: `set DXR_LEIA_LIFT=0` (or rename the NeurD install key) → + `lift caps` reports `modes=0 state=unavailable`, and a normal + `cube_handle_d3d11_win` session weaves exactly as before. +4. Licence path: first activation offline → `state=activating` with a + `LICENSE_NETWORK_ERROR` WARN; reconnect → ready within ~15 s of the next call. + +## Known limits + +- **Fallback path only:** against a runtime without `create_dp_d3d11_lift`, the runtime + calls the ordinary factory with a NULL window, which builds an async SR weaver (it + cannot be told apart from a NULL-HWND `_texture` DP). That extra weaver can briefly + perturb the lens. The lift-only factory removes this on runtimes that have it. +- Windows / D3D11 only. The D3D12, GL and Vulkan DPs do not implement lift. +- The activation thread is detached; unloading the plug-in DLL while it runs + (a first activation in progress) is unsafe. The runtime does not unload plug-ins + mid-session today. +- Caps are frozen once READY: the runtime (`d3d11_lift.cpp`) polls `lift_get_caps` only + while the module is not READY. A convert-time `NEURD_UNAVAILABLE_OUTDATED_RUNTIME` + latch (SBS then falls back to the default pattern) is therefore not reflected in caps. +- NeurD's 32-stream limit is process-wide and shared with anything else in the + service process that uses NeurD. diff --git a/scripts/build-windows.bat b/scripts/build-windows.bat index 5bd4c08..ac9b5be 100644 --- a/scripts/build-windows.bat +++ b/scripts/build-windows.bat @@ -14,6 +14,7 @@ setlocal enabledelayedexpansion :: :: Required environment (auto-fetched if missing): :: LEIASR_SDKROOT — extracted LeiaSR-SDK-*-win64 dir +:: NEURD_SDK_ROOT — dir with include\NeurD.h (+ generated NeurD_version.h); default: fetched :: :: Optional environment: :: DXR_RUNTIME_SOURCE_DIR — path to a local displayxr-runtime checkout. @@ -44,6 +45,17 @@ set SR_TAG=1.35.0.2011 :: Leia's PRIVATE artifact repo -- it is Leia SDK material and must not be :: publicly reachable. `gh auth status` must show an account with read access. set SR_SDK_REPO=LeiaInc/SR-SDK-Windows-Releases-Internal-Public +:: --- NeurD pins (2D->3D lift, docs/lift-neurd.md) ----------------------- +:: The lift module compiles against the REAL NeurD headers, fetched from the +:: PRIVATE LeiaInc/media_sdk repo at NEURD_SDK_REF into the gitignored +:: NeurD-SDK-\include (never committed -- Leia-private, like the SR SDK). +:: Same gh auth as the SR fetch. KEEP IN SYNC with build-windows.yml +:: (scripts/check_sr_pins.py asserts it). Pin a media_sdk RELEASE TAG; v0.4.6 +:: is the newest tag and its headers are byte-identical to dev@25a713d93 (the +:: tip the module was written against). A failed fetch is SOFT: lift compiles +:: out, nothing else changes. +set NEURD_SDK_REF=v0.4.6 +set NEURD_SDK_REPO=LeiaInc/media_sdk :: Stamp-aware (ST-5318) Vulkan weaver for SR 1.36.x. Separate tag because it is :: grafted from a different SR branch than SR_TAG's SDK — see the release notes. set SR_VKSTAMP_TAG=sr-sdk-v1.36.4.17537-vkstamp @@ -208,10 +220,26 @@ if not exist "%LEIASR_V2_SDKROOT%\lib\srSDK_loader.lib" ( echo SR v2 SDK ready. ) +:: --- NeurD headers (optional; soft failure) --- +:: An existing NEURD_SDK_ROOT (e.g. a local media_sdk\sdk drop with a generated +:: NeurD_version.h) takes priority; otherwise fetch the pinned headers. +if "%NEURD_SDK_ROOT%"=="" ( + set NEURD_SDK_ROOT=%REPO%NeurD-SDK-%NEURD_SDK_REF% +) +if not exist "%NEURD_SDK_ROOT%\include\NeurD_version.h" ( + echo === Fetching NeurD headers ^(%NEURD_SDK_REPO%@%NEURD_SDK_REF%^) === + powershell -NoProfile -ExecutionPolicy Bypass -File "%SCRIPT_DIR%fetch-neurd-headers.ps1" -Repo %NEURD_SDK_REPO% -Ref %NEURD_SDK_REF% -Dest "%NEURD_SDK_ROOT%" + if !ERRORLEVEL! NEQ 0 ( + echo WARN: NeurD headers unavailable -- building WITHOUT 2D->3D lift. + set NEURD_SDK_ROOT= + ) +) + echo. echo === Dependencies ready === echo LEIASR_SDKROOT=%LEIASR_SDKROOT% echo LEIASR_V2_SDKROOT=%LEIASR_V2_SDKROOT% +echo NEURD_SDK_ROOT=%NEURD_SDK_ROOT% echo DXR_RUNTIME_SOURCE_DIR=%DXR_RUNTIME_SOURCE_DIR% echo. @@ -256,6 +284,9 @@ REM because the two SDKs are independent drops and the migration needs both REM present at once. Unset simply builds the v1-only plug-in, which is the REM shipping configuration until a win64 v2 drop is generally available. REM set LEIASR_V2_SDKROOT=C:\path\to\LeiaSR-SDK-...-win64-Release +if not "%NEURD_SDK_ROOT%"=="" ( + set CMAKE_ARGS=!CMAKE_ARGS! -DNEURD_SDK_ROOT="%NEURD_SDK_ROOT%" +) if not "%LEIASR_V2_SDKROOT%"=="" ( set CMAKE_ARGS=!CMAKE_ARGS! -DLEIASR_V2_SDKROOT="%LEIASR_V2_SDKROOT%" echo Building WITH the SR v2 C API ^(LEIASR_V2_SDKROOT set^). diff --git a/scripts/check_sr_pins.py b/scripts/check_sr_pins.py index 4016cb4..5df8694 100644 --- a/scripts/check_sr_pins.py +++ b/scripts/check_sr_pins.py @@ -67,7 +67,10 @@ LINUX_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build-linux.yml" # The five pins that must be identical in the .bat and in build-windows.yml. -PINS = ("SR_TAG", "SR_VKSTAMP_TAG", "SR_V2_TAG", "SR_V2_DIR", "SR_SDK_REPO") +# NEURD_SDK_REF / NEURD_SDK_REPO pin the private NeurD headers the 2D->3D lift +# module is built against (docs/lift-neurd.md) -- same two-file duplication, +# same drift hazard, so they ride the same check. +PINS = ("SR_TAG", "SR_VKSTAMP_TAG", "SR_V2_TAG", "SR_V2_DIR", "SR_SDK_REPO", "NEURD_SDK_REF", "NEURD_SDK_REPO") # sr-sdk-v2-1.37.0.1502 -> ("1.37.0", "1502") SR_V2_TAG_RE = re.compile(r"^sr-sdk-v2-(\d+\.\d+\.\d+)\.(\d+)$") diff --git a/scripts/fetch-neurd-headers.ps1 b/scripts/fetch-neurd-headers.ps1 new file mode 100644 index 0000000..f4df3ca --- /dev/null +++ b/scripts/fetch-neurd-headers.ps1 @@ -0,0 +1,64 @@ +# Copyright 2026, Leia Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# Fetch the NeurD public headers from the PRIVATE LeiaInc/media_sdk repo at a +# pinned ref, for the plug-in's 2D->3D lift module (docs/lift-neurd.md). +# +# These headers are Leia-private: they land in a gitignored directory +# (NeurD-SDK-/include, see .gitignore) and must never be committed or +# re-hosted on this public repo -- same rule as the SR SDK. +# +# Auth: `gh` with an account (or GH_TOKEN) that can read LeiaInc/media_sdk. +# CI passes secrets.LEIALOFT_GITHUB_TOKEN, the token the SR SDK fetch uses. +# +# Exit 0 = headers ready; 1 = fetch failed. Callers treat failure as SOFT: +# the lift module compiles out and the rest of the plug-in builds unchanged. +# +# Usage: fetch-neurd-headers.ps1 -Repo LeiaInc/media_sdk -Ref v0.4.6 -Dest + +param( + [Parameter(Mandatory = $true)][string]$Repo, + [Parameter(Mandatory = $true)][string]$Ref, + [Parameter(Mandatory = $true)][string]$Dest +) + +$ErrorActionPreference = 'Stop' +$inc = Join-Path $Dest 'include' + +function Get-RepoFile([string]$path) { + # Raw file contents at the pinned ref. gh exits non-zero on 404 / no auth. + $out = & gh api "repos/$Repo/contents/${path}?ref=$Ref" -H 'Accept: application/vnd.github.raw' 2>&1 + if ($LASTEXITCODE -ne 0) { throw "gh api $path@$Ref failed: $out" } + return ($out -join "`n") + "`n" +} + +try { + New-Item -ItemType Directory -Force $inc | Out-Null + + # NeurD.h includes NeurD_version.h and _NeurD_detail.h; the latter includes + # _NeurD_table.h. That is the whole closure. + foreach ($f in @('NeurD.h', '_NeurD_detail.h', '_NeurD_table.h')) { + Set-Content -Path (Join-Path $inc $f) -Value (Get-RepoFile "sdk/$f") -NoNewline -Encoding UTF8 + } + + # NeurD_version.h is generated by media_sdk's CMake from NeurD_version.h.in + # and the top-level project(... VERSION "x.y.z"). Reproduce that exactly. + $top = Get-RepoFile 'CMakeLists.txt' + $m = [regex]::Match($top, 'project\s*\(\s*mediasdk[^)]*VERSION\s+"?(\d+)\.(\d+)\.(\d+)"?') + if (-not $m.Success) { throw "could not find project(mediasdk ... VERSION x.y.z) in CMakeLists.txt@$Ref" } + $ver = (Get-RepoFile 'sdk/NeurD_version.h.in'). + Replace('@PROJECT_VERSION_MAJOR@', $m.Groups[1].Value). + Replace('@PROJECT_VERSION_MINOR@', $m.Groups[2].Value). + Replace('@PROJECT_VERSION_PATCH@', $m.Groups[3].Value) + Set-Content -Path (Join-Path $inc 'NeurD_version.h') -Value $ver -NoNewline -Encoding UTF8 + + Write-Host "NeurD headers $($m.Groups[1].Value).$($m.Groups[2].Value).$($m.Groups[3].Value) ($Repo@$Ref) -> $inc" + exit 0 +} catch { + Write-Host "WARNING: NeurD header fetch failed: $_" + Write-Host " 2D->3D lift will be compiled OUT (plug-in otherwise unchanged)." + Write-Host " Needs gh auth with read access to $Repo (gh auth login / GH_TOKEN)." + # Do not leave a half-populated include dir that CMake would accept. + if (Test-Path $inc) { Remove-Item -Recurse -Force $inc } + exit 1 +} diff --git a/scripts/set-lift-knobs.bat b/scripts/set-lift-knobs.bat new file mode 100644 index 0000000..d35ccaf --- /dev/null +++ b/scripts/set-lift-knobs.bat @@ -0,0 +1,47 @@ +@echo off +setlocal + +:: ============================================================ +:: Set the Leia lift demo knobs in the registry (run ELEVATED). +:: ============================================================ +:: Usage: scripts\set-lift-knobs.bat write Backend=directml, InteractiveMin=0.4.4 +:: scripts\set-lift-knobs.bat --clear delete HKLM\SOFTWARE\DisplayXR\Leia\Lift +:: +:: The lift reads each knob from the env var first (DXR_LEIA_LIFT_*), else from +:: this key, else its default. The registry survives displayxr-service.exe +:: respawns (tray relaunch, HKLM Run at logon, crash restart), which start with +:: the logon environment and would otherwise drop env-only knobs silently. +:: InteractiveMin=0.4.4 is ONLY for the NeurD 0.4.4 internal-interactive package; +:: on a stock NeurD 0.4.4 it crashes the service. See docs\lift-neurd.md. +:: Restart displayxr-service.exe afterwards for the change to take effect. + +set "KEY=HKLM\SOFTWARE\DisplayXR\Leia\Lift" + +net session >nul 2>&1 +if errorlevel 1 ( + echo ERROR: run this from an elevated ^(Administrator^) prompt. + exit /b 1 +) + +if /i "%~1"=="--clear" ( + reg query "%KEY%" /reg:64 >nul 2>&1 + if errorlevel 1 ( + echo %KEY% not present - nothing to clear. + exit /b 0 + ) + reg delete "%KEY%" /f /reg:64 || exit /b 1 + echo Deleted %KEY%. + exit /b 0 +) + +if not "%~1"=="" ( + echo Usage: %~nx0 [--clear] + exit /b 2 +) + +reg add "%KEY%" /v Backend /t REG_SZ /d directml /f /reg:64 || exit /b 1 +reg add "%KEY%" /v InteractiveMin /t REG_SZ /d 0.4.4 /f /reg:64 || exit /b 1 +echo Wrote %KEY%: Backend=directml InteractiveMin=0.4.4 +reg query "%KEY%" /reg:64 +echo Restart displayxr-service.exe for the lift to pick these up. +exit /b 0 diff --git a/src/drv_leia/CMakeLists.txt b/src/drv_leia/CMakeLists.txt index b8348d3..8458b8b 100644 --- a/src/drv_leia/CMakeLists.txt +++ b/src/drv_leia/CMakeLists.txt @@ -198,6 +198,33 @@ if(EXISTS "${SR_PATH}/lib/SimulatedRealityOpenGL.lib") set(LEIA_SR_GL_AVAILABLE TRUE) endif() +# --- NeurD 2D->3D lift (optional, additive) --------------------------------- +# docs/lift-neurd.md. The module compiles against the REAL NeurD headers, which +# are Leia-private and are never committed here: scripts/build-windows.bat and +# build-windows.yml fetch them from LeiaInc/media_sdk at NEURD_SDK_REF (pinned +# next to the SR pins) into a gitignored NeurD-SDK-/include. NeurD.dll is +# still loaded dynamically at first use — no import lib, no link dependency. +# Headers absent (no token / offline / fork PR) => the module compiles OUT, the +# DP's lift slots stay NULL, the runtime reports lift unavailable, and the rest +# of the plug-in builds exactly as before. Never a hard failure. +set(LEIA_NEURD_AVAILABLE FALSE) +if(NOT DEFINED NEURD_SDK_ROOT AND DEFINED ENV{NEURD_SDK_ROOT}) + set(NEURD_SDK_ROOT "$ENV{NEURD_SDK_ROOT}") +endif() +if(LEIA_SR_D3D11_AVAILABLE AND NEURD_SDK_ROOT + AND EXISTS "${NEURD_SDK_ROOT}/include/NeurD.h" + AND EXISTS "${NEURD_SDK_ROOT}/include/NeurD_version.h" + AND EXISTS "${NEURD_SDK_ROOT}/include/_NeurD_detail.h" + AND EXISTS "${NEURD_SDK_ROOT}/include/_NeurD_table.h") + set(LEIA_NEURD_AVAILABLE TRUE) + list(APPEND LEIA_SOURCES leia_lift_neurd.cpp leia_lift_neurd.h) + message(STATUS "NeurD headers found at ${NEURD_SDK_ROOT} - enabling the D3D11 2D->3D lift slots") +elseif(LEIA_SR_D3D11_AVAILABLE) + message(STATUS "NeurD headers NOT found (NEURD_SDK_ROOT='${NEURD_SDK_ROOT}') - 2D->3D lift compiled OUT; " + "the plug-in is otherwise unchanged. Fetch them with scripts/build-windows.bat " + "(needs gh auth with read access to LeiaInc/media_sdk).") +endif() + # Windows Graphics Capture for compose-under-bg transparency. The GL DP imports # the captured D3D11 desktop texture into GL via WGL_NV_DX_interop2, so it needs # this module too. @@ -340,6 +367,11 @@ if(LEIA_SR_D3D11_AVAILABLE) target_compile_definitions(DisplayXR-LeiaSR PRIVATE XRT_HAVE_LEIA_SR_D3D11) endif() +if(LEIA_NEURD_AVAILABLE) + target_include_directories(DisplayXR-LeiaSR PRIVATE "${NEURD_SDK_ROOT}/include") + target_compile_definitions(DisplayXR-LeiaSR PRIVATE DXR_LEIA_HAS_NEURD) +endif() + if(LEIA_SR_D3D12_AVAILABLE) target_link_libraries(DisplayXR-LeiaSR PRIVATE d3d12 d3dcompiler) target_compile_definitions(DisplayXR-LeiaSR PRIVATE XRT_HAVE_LEIA_SR_D3D12) diff --git a/src/drv_leia/leia_display_processor_d3d11.cpp b/src/drv_leia/leia_display_processor_d3d11.cpp index 0054843..7ecd1d1 100644 --- a/src/drv_leia/leia_display_processor_d3d11.cpp +++ b/src/drv_leia/leia_display_processor_d3d11.cpp @@ -52,6 +52,22 @@ #define DXR_LEIA_DP_D3D11_BACKEND_STATE 1 #endif +/* + * 2D->3D lift (runtime feat/lift-ext, XRT_DP_D3D11_HAS_LIFT) — four appended + * slots backed by NeurD (leia_lift_neurd.cpp, docs/lift-neurd.md). Same + * coupled-addition pattern: against older runtime headers the slots compile out + * and struct_size tells the runtime they are absent. The NeurD module also + * needs the real NeurD headers, fetched at build time from the private + * media_sdk repo (DXR_LEIA_HAS_NEURD); without them the slots stay NULL and the + * runtime reports lift unavailable. Even when built, NeurD.dll is loaded only + * when a lift slot is first called. + */ +#if defined(XRT_DP_D3D11_HAS_LIFT) && defined(DXR_LEIA_HAS_NEURD) +#define DXR_LEIA_DP_D3D11_LIFT 1 +#include "leia_lift_neurd.h" +#include // offsetof +#endif + // Fullscreen quad vertex shader (4 vertices, triangle strip via SV_VertexID) static const char *blit_vs_source = R"( @@ -513,6 +529,12 @@ struct leia_display_processor_d3d11_impl //! it over the live desktop. NULL ⟹ no backdrop (desktop-only). ID3D11ShaderResourceView *backdrop_srv; //!< NOT owned (compositor-owned). uint32_t backdrop_w, backdrop_h; + +#ifdef DXR_LEIA_DP_D3D11_LIFT + //! NeurD 2D->3D lift handle (owned; created at factory time — cheap, no + //! DLL load — destroyed in leia_dp_d3d11_destroy). NULL only on OOM. + struct leia_lift_neurd *lift; +#endif }; static inline struct leia_display_processor_d3d11_impl * @@ -2287,6 +2309,122 @@ leia_dp_d3d11_set_window(struct xrt_display_processor_d3d11 *xdp, void *window_h } #endif +#ifdef DXR_LEIA_DP_D3D11_LIFT +/* + * 2D->3D lift slots. Thin translation of the runtime's xrt_dp_lift_* contract + * onto leia_lift_neurd; every size-versioned struct is read only as far as its + * struct_size covers (ADR-020 append-only rule), so a runtime that grows these + * structs keeps working against this plug-in and vice versa. + */ +static_assert(LEIA_LIFT_MODE_DEPTH == XRT_DP_LIFT_MODE_DEPTH && LEIA_LIFT_MODE_SBS == XRT_DP_LIFT_MODE_SBS && + LEIA_LIFT_MODE_NVIEW == XRT_DP_LIFT_MODE_NVIEW, + "lift mode bits must match the runtime's XRT_DP_LIFT_MODE_*"); +static_assert(LEIA_LIFT_STATE_UNAVAILABLE == XRT_DP_LIFT_STATE_UNAVAILABLE && + LEIA_LIFT_STATE_ACTIVATING == XRT_DP_LIFT_STATE_ACTIVATING && + LEIA_LIFT_STATE_READY == XRT_DP_LIFT_STATE_READY, + "lift states must match the runtime's XRT_DP_LIFT_STATE_*"); + +#define LEIA_LIFT_COVERS(ptr, type, field) \ + ((ptr)->struct_size >= offsetof(type, field) + sizeof(((type *)0)->field)) + +static bool +leia_dp_d3d11_lift_get_caps(struct xrt_display_processor_d3d11 *xdp, struct xrt_dp_lift_caps *out) +{ + struct leia_display_processor_d3d11_impl *ldp = leia_dp_d3d11(xdp); + if (out == NULL || out->struct_size < offsetof(struct xrt_dp_lift_caps, state) + sizeof(uint32_t)) { + return false; + } + struct leia_lift_neurd_caps c = {}; + if (!leia_lift_neurd_get_caps(ldp->lift, &c)) { + return false; + } + struct xrt_dp_lift_caps full = {}; + full.struct_size = out->struct_size; + full.modes = c.modes; + full.max_streams = c.max_streams; + full.max_views = c.max_views; + full.depth_semantics = c.depth_semantics; + full.state = c.state; + full.typical_latency_ns = c.typical_latency_ns; + static_assert(sizeof(full.backend) <= sizeof(c.backend), "backend name buffers"); + memcpy(full.backend, c.backend, sizeof(full.backend)); + full.backend[sizeof(full.backend) - 1] = '\0'; + memcpy(out, &full, out->struct_size < sizeof(full) ? out->struct_size : sizeof(full)); + return true; +} + +static bool +leia_dp_d3d11_lift_stream_create(struct xrt_display_processor_d3d11 *xdp, + const struct xrt_dp_lift_stream_info *info, + uint64_t *out_id) +{ + struct leia_display_processor_d3d11_impl *ldp = leia_dp_d3d11(xdp); + if (info == NULL || out_id == NULL || !LEIA_LIFT_COVERS(info, struct xrt_dp_lift_stream_info, mode)) { + return false; + } + struct leia_lift_neurd_stream_desc d = {}; + d.mode = info->mode; + d.content_hint = LEIA_LIFT_COVERS(info, struct xrt_dp_lift_stream_info, content_hint) ? info->content_hint : 0; + d.input_scale = LEIA_LIFT_COVERS(info, struct xrt_dp_lift_stream_info, input_scale) ? info->input_scale : 0.0f; + return leia_lift_neurd_stream_create(ldp->lift, &d, out_id); +} + +static void +leia_dp_d3d11_lift_stream_destroy(struct xrt_display_processor_d3d11 *xdp, uint64_t id) +{ + leia_lift_neurd_stream_destroy(leia_dp_d3d11(xdp)->lift, id); +} + +static bool +leia_dp_d3d11_lift_convert(struct xrt_display_processor_d3d11 *xdp, + uint64_t id, + void *d3d11_context, + void *input_resource, + uint32_t w, + uint32_t h, + const struct xrt_dp_lift_params *p, + const float *viewpoints_xyz, + uint32_t viewpoint_floats, + void **out_resource, + uint32_t *out_w, + uint32_t *out_h, + uint32_t *out_format) +{ + struct leia_display_processor_d3d11_impl *ldp = leia_dp_d3d11(xdp); + + struct leia_lift_neurd_params lp = {-1.0f, 1.0f, 0, 2}; + if (p != NULL) { + if (LEIA_LIFT_COVERS(p, struct xrt_dp_lift_params, convergence)) { + lp.convergence = p->convergence; + } + if (LEIA_LIFT_COVERS(p, struct xrt_dp_lift_params, strength)) { + lp.strength = p->strength; + } + if (LEIA_LIFT_COVERS(p, struct xrt_dp_lift_params, inpaint)) { + lp.inpaint = p->inpaint; + } + if (LEIA_LIFT_COVERS(p, struct xrt_dp_lift_params, view_count)) { + lp.view_count = p->view_count; + } + } + + // Tracked eyes (metres, display space) — the same source the runtime's + // get_predicted_eye_positions slot reads; the wrapper is exception-safe. + // Only consulted when the runtime did not pass explicit viewpoints. + float left[3] = {0, 0, 0}, right[3] = {0, 0, 0}; + bool eyes_valid = false; + if ((viewpoints_xyz == NULL || viewpoint_floats == 0) && ldp->leiasr != NULL && + leiasr_d3d11_get_predicted_eye_positions(ldp->leiasr, left, right)) { + const float dx = right[0] - left[0], dy = right[1] - left[1], dz = right[2] - left[2]; + eyes_valid = (dx * dx + dy * dy + dz * dz) > 1e-6f; // same tracking-loss test as the eye slot + } + + return leia_lift_neurd_convert(ldp->lift, id, d3d11_context, input_resource, w, h, &lp, + (viewpoint_floats > 0) ? viewpoints_xyz : NULL, viewpoint_floats, left, right, + eyes_valid, out_resource, out_w, out_h, out_format); +} +#endif // DXR_LEIA_DP_D3D11_LIFT + static void leia_dp_d3d11_destroy(struct xrt_display_processor_d3d11 *xdp) { @@ -2305,6 +2443,12 @@ leia_dp_d3d11_destroy(struct xrt_display_processor_d3d11 *xdp) ldp->blit_cb->Release(); } +#ifdef DXR_LEIA_DP_D3D11_LIFT + // Before the weaver: lift streams hold bridge resources on the service + // device; the NeurD instance itself stays up for the process (see module). + leia_lift_neurd_destroy(&ldp->lift); +#endif + compose_release_resources(ldp); ck_release_resources(ldp); @@ -2416,6 +2560,20 @@ leia_dp_d3d11_init_vtable(struct leia_display_processor_d3d11_impl *ldp) #ifdef DXR_LEIA_DP_D3D11_BACKEND_STATE ldp->base.get_backend_state = leia_dp_d3d11_get_backend_state; // #158 SR restart self-heal (slot 21) #endif +#ifdef DXR_LEIA_DP_D3D11_LIFT + ldp->base.lift_get_caps = leia_dp_d3d11_lift_get_caps; // 2D->3D lift (NeurD) + ldp->base.lift_stream_create = leia_dp_d3d11_lift_stream_create; + ldp->base.lift_stream_destroy = leia_dp_d3d11_lift_stream_destroy; + ldp->base.lift_convert = leia_dp_d3d11_lift_convert; + ldp->lift = leia_lift_neurd_create(); // reads DXR_LEIA_LIFT* once; loads nothing + U_LOG_W("Leia D3D11 DP: lift slots WIRED (NeurD, loaded on first use)"); +#else +#if !defined(XRT_DP_D3D11_HAS_LIFT) + U_LOG_W("Leia D3D11 DP: lift slots NOT COMPILED (runtime headers predate XRT_DP_D3D11_HAS_LIFT)"); +#else + U_LOG_W("Leia D3D11 DP: lift slots NOT COMPILED (NeurD headers were not fetched at build time)"); +#endif +#endif } @@ -2585,6 +2743,58 @@ leia_dp_factory_d3d11(void *d3d11_device, } +#ifdef DXR_LEIA_DP_D3D11_LIFT +/* + * + * Lift-only factory — xrt_plugin_iface::create_dp_d3d11_lift (ADR-042). + * + * The runtime creates exactly one of these per process on a dedicated device + * and drives ONLY the lift_* slots on it, from its lift thread, always with + * explicit viewpoints. So: no SR weaver (a second weaver on the shared SR + * context perturbs the lens), no window, no tracker, no lens control, no blit + * shaders — just the NeurD lift handle. Every non-lift slot stays NULL, which + * the runtime's slot helpers read as "absent". NeurD itself is still loaded + * lazily, on the first lift call. + * + */ + +extern "C" xrt_result_t +leia_dp_factory_d3d11_lift(void *d3d11_device, + void *d3d11_context, + void *window_handle, + struct xrt_display_processor_d3d11 **out_xdp) +{ + (void)d3d11_context; + (void)window_handle; // always NULL for the lift DP + if (out_xdp == NULL) { + return XRT_ERROR_DEVICE_CREATION_FAILED; + } + struct leia_display_processor_d3d11_impl *ldp = + (struct leia_display_processor_d3d11_impl *)calloc(1, sizeof(*ldp)); + if (ldp == NULL) { + return XRT_ERROR_ALLOCATION; + } + ldp->base.struct_size = static_cast(sizeof(struct xrt_display_processor_d3d11)); + ldp->base.destroy = leia_dp_d3d11_destroy; // NULL-safe for every member left unset here + ldp->base.lift_get_caps = leia_dp_d3d11_lift_get_caps; + ldp->base.lift_stream_create = leia_dp_d3d11_lift_stream_create; + ldp->base.lift_stream_destroy = leia_dp_d3d11_lift_stream_destroy; + ldp->base.lift_convert = leia_dp_d3d11_lift_convert; + // lift_convert_blob (GAUSSIANS) stays NULL: NeurD has no splat path. + ldp->device = static_cast(d3d11_device); + ldp->lift = leia_lift_neurd_create(); // reads DXR_LEIA_LIFT* once; loads nothing + ldp->view_count = 2; + ldp->async_zone_publish = true; + // leiasr stays NULL: lift_convert then uses only the runtime's explicit + // viewpoints (or NeurD's default pattern) — never a tracker. + + *out_xdp = &ldp->base; + U_LOG_W("Created Leia D3D11 LIFT-ONLY display processor (no weaver, no window; NeurD loaded on first use)"); + return XRT_SUCCESS; +} +#endif // DXR_LEIA_DP_D3D11_LIFT + + /* * * Legacy creation function — wraps an existing leiasr_d3d11 handle. diff --git a/src/drv_leia/leia_display_processor_d3d11.h b/src/drv_leia/leia_display_processor_d3d11.h index 7a74691..41c1927 100644 --- a/src/drv_leia/leia_display_processor_d3d11.h +++ b/src/drv_leia/leia_display_processor_d3d11.h @@ -35,6 +35,19 @@ leia_dp_factory_d3d11(void *d3d11_device, void *window_handle, struct xrt_display_processor_d3d11 **out_xdp); +#if defined(DXR_LEIA_HAS_NEURD) +/*! + * Lift-only D3D11 DP (xrt_plugin_iface::create_dp_d3d11_lift, ADR-042): no + * weaver, no window, only the NeurD lift_* slots. Defined only when the + * runtime headers carry the lift slots AND the NeurD headers were fetched. + */ +xrt_result_t +leia_dp_factory_d3d11_lift(void *d3d11_device, + void *d3d11_context, + void *window_handle, + struct xrt_display_processor_d3d11 **out_xdp); +#endif + /*! * Create an @ref xrt_display_processor_d3d11 that wraps a Leia SR SDK * D3D11 weaver (leiasr_d3d11_set_input_texture + leiasr_d3d11_weave). diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp new file mode 100644 index 0000000..9a45340 --- /dev/null +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -0,0 +1,1955 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: Apache-2.0 +/*! + * @file + * @brief NeurD-backed 2D->3D conversion ("lift") for the Leia D3D11 DP. + * + * Design notes (full write-up: docs/lift-neurd.md): + * + * - Dynamic load only. NeurD.dll is located the way NeurD's own loader does it + * (PATH -> NEURD_PATH -> HKLM\SOFTWARE\LeiaInc\NeurD default value) but it + * is not a link dependency: a machine without NeurD loads this plug-in + * exactly as before and lift reports unavailable. + * + * - One NeurD instance per process (NeurD's rule). Loaded + initialised on a + * detached background thread, never on a caller's thread: NeurD_init runs a + * licence activation that needs the network on first use. A network failure + * is reported as ACTIVATING and retried on later calls, rate-limited. + * NeurD is never de-initialised once up: DP instances come and go on focus + * changes, and re-init re-runs licensing + model load. The ref-count only + * decides when to shrink NeurD's memory pool. + * + * - Device bridge (the LeiaMeet DxStereoConverter pattern). NeurD runs on ITS + * OWN D3D11 device (NeurD_get_dx_device, 0.4.3+), possibly on a different adapter + * than the runtime's service device on hybrid boxes. NeurD's DX input must + * be a raw RGBA8 ID3D11Buffer with D3D11_RESOURCE_MISC_SHARED, and NeurD + * hands its output back on the input buffer's device. Buffers do not share + * reliably across devices, textures do — so all buffers live on NeurD's + * device, and the crossing is done with legacy-shared TEXTURES created on + * NeurD's device and opened on ours: + * + * ours: input tex --CopySubresourceRegion--> in_bridge (opened) + * CPU: event-query drain on our context + * NeurD: in_bridge SRV --pack CS--> raw input buffer ; drain + * NeurD: convert_stream_dx[_interactive] (blocking) + * NeurD: output buffer --unpack CS--> out_bridge UAV ; drain + * ours: out_bridge (opened) is returned to the runtime + * + * Every hop is CPU-drained with an event query: NeurD's DirectML work runs + * on its own queue, unordered against any D3D11 context. + * + * NeurD 0.3.11-0.4.2 has the stream API but no get_dx_device. There the + * plug-in creates the "NeurD device" itself (default adapter, like the SDK's + * own example) and owns it; NeurD 0.3.x's CUDA DX path runs its D3D11-CUDA + * interop on whatever device the input buffer lives on and hands back an + * RGBA8 texture on that same device, so the bridge is unchanged. + * + * - NeurD properties are process-global, so the (set props -> convert) pair + * must be atomic across streams: one process-wide mutex serialises every + * NeurD call and every use of NeurD's immediate context. + * + * - Never throws, never blocks the caller on activation; errors return false + * with a WARN once per site. + */ + +#include "leia_lift_neurd.h" +// The REAL NeurD header, fetched at build time from the private LeiaInc/media_sdk +// repo at the pinned NEURD_SDK_REF (never committed here; see docs/lift-neurd.md). +// Only its types, PFN typedefs and header-inline table wrappers are used — the +// wrappers dispatch through the table NeurD_load returns and version-check each +// entry — so there is no import lib and no link dependency on NeurD, and any +// NeurD signature drift is a compile error in this file. +#include + +#include "util/u_logging.h" + +#include +#include +#include // ID3D11Multithread +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +//! Table entry @p name is present in the LOADED runtime (version test first, so +//! an older, physically shorter table is never read past its end). The +//! NEURD__INTRODUCED_IN constants come from the real header. +#define LEIA_NEURD_HAS(nd, name) ((nd) != nullptr && (nd)->version >= NEURD_##name##_INTRODUCED_IN && (nd)->name != nullptr) + +namespace { + +/* + * + * Constants + small helpers. + * + */ + +//! Reference inter-pupillary distance: NeurD's default stereo pattern places +//! the two views at x = -0.5 / +0.5, i.e. ONE unit of x == one "nominal eye +//! baseline". Mapping metres through this IPD makes a centred, tracked viewer +//! reproduce the default pattern exactly. +constexpr float kIpdRefM = 0.063f; +//! Clamp for mapped viewpoints — well past any comfortable look-around; stops a +//! tracker glitch from asking NeurD for a wildly extrapolated view. +constexpr float kViewpointClamp = 3.0f; +//! NEURD_PROP_GAIN_MULTIPLIER at strength 1 (DepthGain knob). Panel-calibrated 2026-09-26. +constexpr float kDefaultDepthGain = 2.0f; +//! NEURD_PROP_DILATE_RADIO (Dilate knob). NeurD's own default is 3; the foreground grew visibly +//! past its silhouette on the panel (David, 2026-09-26: "the disparity map dilation is a bit too +//! much, reduce a bit"). +constexpr int32_t kDefaultDilate = 2; +//! NeurD's MAX_STREAMS. +constexpr uint32_t kMaxStreams = 32; +//! N views go side by side in ONE row (runtime contract), so the output is +//! N x inference-width wide; 8 x 2560 (1440p) still fits D3D11's 16384 limit. +constexpr uint32_t kMaxViews = 8; +constexpr uint32_t kMaxTexDim = D3D11_REQ_TEXTURE2D_U_OR_V_DIMENSION; +//! Retry period for a licence activation that failed on the network. +constexpr uint64_t kActivationRetryMs = 15000; +//! Upper bound for any CPU drain of a GPU queue (device-lost guard). +constexpr uint64_t kDrainTimeoutMs = 2000; +//! Latency priors (reported until a measurement exists). +constexpr uint64_t kPriorLatencyDirectMlNs = 22000000ull; +constexpr uint64_t kPriorLatencyCudaNs = 14000000ull; +constexpr uint64_t kPriorLatencyOtherNs = 40000000ull; + +#define LIFT_WARN_ONCE(...) \ + do { \ + static std::atomic warned_{false}; \ + if (!warned_.exchange(true)) { \ + U_LOG_W(__VA_ARGS__); \ + } \ + } while (0) + +template +void +safe_release(T *&p) +{ + if (p != nullptr) { + p->Release(); + p = nullptr; + } +} + +inline float +clampf(float v, float lo, float hi) +{ + return v < lo ? lo : (v > hi ? hi : v); +} + +const char * +status_str(enum NeurD_status s) +{ + switch (s) { + case NEURD_SUCCESS: return "SUCCESS"; + case NEURD_IN_PROGRESS: return "IN_PROGRESS"; + case NEURD_GENERIC_ERROR: return "GENERIC_ERROR"; + case NEURD_INVALID_ARG: return "INVALID_ARG"; + case NEURD_INVALID_LICENSE: return "INVALID_LICENSE"; + case NEURD_UNAVAILABLE_OUTDATED_RUNTIME: return "UNAVAILABLE_OUTDATED_RUNTIME"; + case NEURD_NOT_INITIALIZED: return "NOT_INITIALIZED"; + case NEURD_LICENSE_NETWORK_ERROR: return "LICENSE_NETWORK_ERROR"; + case NEURD_INVALID_MODEL: return "INVALID_MODEL"; + case NEURD_FILESYSTEM_ERROR: return "FILESYSTEM_ERROR"; + case NEURD_NETWORK_ERROR: return "NETWORK_ERROR"; + case NEURD_ABORTED: return "ABORTED"; + default: return "UNKNOWN"; + } +} + +const char * +backend_str(enum NeurD_backend b) +{ + switch (b) { + case NEURD_BACKEND_CUDA: return "cuda"; + case NEURD_BACKEND_DIRECTML: return "directml"; + case NEURD_BACKEND_OPENVINO: return "openvino"; + default: return "unknown"; + } +} + +/* + * + * Knobs (read once per DP handle, at leia_lift_neurd_create). Each comes from + * the environment, else HKLM\SOFTWARE\DisplayXR\Leia\Lift (REG_SZ, same + * grammar), else its default. The registry exists because a respawned service + * (tray relaunch, HKLM Run at logon, crash restart) starts with the logon env + * and would otherwise lose env-only knobs silently. + * + */ + +enum backend_choice +{ + BACKEND_AUTO = -1, + BACKEND_CUDA = NEURD_BACKEND_CUDA, + BACKEND_DIRECTML = NEURD_BACKEND_DIRECTML, + BACKEND_OPENVINO = NEURD_BACKEND_OPENVINO, +}; + +enum knob_src +{ + KSRC_DEFAULT = 0, + KSRC_ENV, + KSRC_REG, +}; + +const char * +knob_src_str(knob_src s) +{ + switch (s) { + case KSRC_ENV: return "env"; + case KSRC_REG: return "reg"; + default: return "default"; + } +} + +struct knobs +{ + bool enabled; //!< DXR_LEIA_LIFT (default on; env only) + int backend; //!< DXR_LEIA_LIFT_BACKEND / Backend (default directml) + int32_t autoscaling; //!< DXR_LEIA_LIFT_SCALE / Scale (default 720p) + bool scale_forced; //!< Scale was set: overrides the stream's input_scale + float view_gain; //!< DXR_LEIA_LIFT_VIEW_GAIN / ViewGain (default 1.0) + float conv_gain; //!< DXR_LEIA_LIFT_CONV_GAIN / ConvGain (default 0.4): relative convergence -> NeurD units + uint64_t interactive_min; //!< DXR_LEIA_LIFT_INTERACTIVE_MIN / InteractiveMin; 0 = unset (header's INTRODUCED_IN) + int32_t video_model; //!< DXR_LEIA_LIFT_VIDEO_MODEL / VideoModel: fast (default) | metric (NeurD >= 0.4.6) + float depth_gain; //!< DXR_LEIA_LIFT_DEPTH_GAIN / DepthGain (default 2.0): NeurD gain at strength 1 + int32_t dilate; //!< DXR_LEIA_LIFT_DILATE / Dilate (default 2): NEURD_PROP_DILATE_RADIO, px + knob_src src_backend, src_scale, src_view_gain, src_conv_gain, src_interactive_min, src_video_model, src_depth_gain, + src_dilate; +}; + +bool +env_ieq(const char *a, const char *b) +{ + return _stricmp(a, b) == 0; +} + +const wchar_t *const kLiftRegKey = L"SOFTWARE\\DisplayXR\\Leia\\Lift"; + +/*! + * One knob's raw string: env @p env_name if set and non-empty, else REG_SZ + * @p reg_name under @p reg (may be NULL), else absent. @p out is UTF-8. + */ +knob_src +knob_lookup(HKEY reg, const char *env_name, const wchar_t *reg_name, char *out, size_t cap) +{ + out[0] = '\0'; + const char *e = std::getenv(env_name); + if (e != nullptr && e[0] != '\0') { + snprintf(out, cap, "%s", e); + return KSRC_ENV; + } + if (reg == nullptr) { + return KSRC_DEFAULT; + } + wchar_t w[128] = {}; + DWORD cb = sizeof(w); // bytes, incl. the terminator RegGetValueW guarantees + LSTATUS r = RegGetValueW(reg, nullptr, reg_name, RRF_RT_REG_SZ, nullptr, w, &cb); + if (r == ERROR_FILE_NOT_FOUND) { + return KSRC_DEFAULT; + } + if (r != ERROR_SUCCESS) { + U_LOG_W("Leia lift: HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift\\%ls unreadable (error %ld, REG_SZ <= 127 chars " + "expected) — ignored", + reg_name, (long)r); + return KSRC_DEFAULT; + } + if (w[0] == L'\0') { + return KSRC_DEFAULT; + } + int n = WideCharToMultiByte(CP_UTF8, 0, w, -1, out, (int)cap, nullptr, nullptr); + if (n <= 0) { + out[0] = '\0'; + return KSRC_DEFAULT; + } + return KSRC_REG; +} + +// ---- One grammar per knob, shared by both sources. Each returns false on a +// value it does not recognise (caller WARNs and keeps the default). + +bool +parse_backend(const char *v, int *out) +{ + if (env_ieq(v, "auto")) { + *out = BACKEND_AUTO; + } else if (env_ieq(v, "cuda")) { + *out = BACKEND_CUDA; + } else if (env_ieq(v, "directml") || env_ieq(v, "dml")) { + *out = BACKEND_DIRECTML; + } else if (env_ieq(v, "openvino")) { + *out = BACKEND_OPENVINO; + } else { + return false; + } + return true; +} + +bool +parse_scale(const char *v, int32_t *out) +{ + if (env_ieq(v, "none") || env_ieq(v, "native") || env_ieq(v, "0")) { + *out = NEURD_INPUT_AUTOSCALING_NONE; + } else if (env_ieq(v, "720") || env_ieq(v, "720p")) { + *out = NEURD_INPUT_AUTOSCALING_720P; + } else if (env_ieq(v, "1080") || env_ieq(v, "1080p")) { + *out = NEURD_INPUT_AUTOSCALING_1080P; + } else if (env_ieq(v, "1440") || env_ieq(v, "1440p")) { + *out = NEURD_INPUT_AUTOSCALING_1440P; + } else { + return false; + } + return true; +} + +bool +parse_float_in(const char *v, float lo, float hi, float *out) +{ + char *end = nullptr; + float f = std::strtof(v, &end); + if (end == v || *end != '\0' || !std::isfinite(f) || f < lo || f > hi) { + return false; + } + *out = f; + return true; +} + +//! Strict "MAJ.MIN.PAT" (no sscanf: MSVC C4996), clamped to >= 0.4.4. +bool +parse_interactive_min(const char *v, uint64_t *out) +{ + unsigned long part[3] = {0, 0, 0}; + const char *c = v; + bool ok = true; + for (int i = 0; i < 3 && ok; i++) { + char *end = nullptr; + part[i] = std::strtoul(c, &end, 10); + ok = end != c && *end == (i < 2 ? '.' : '\0'); + c = end + (i < 2 ? 1 : 0); + } + if (!ok) { + return false; + } + const uint64_t floor_v = NEURD_MAKE_VERSION(0, 4, 4); + const uint64_t ver = NEURD_MAKE_VERSION(part[0], part[1], part[2]); + *out = (ver < floor_v) ? floor_v : ver; + return true; +} + +const char * +scale_str(int32_t a) +{ + switch (a) { + case NEURD_INPUT_AUTOSCALING_NONE: return "none"; + case NEURD_INPUT_AUTOSCALING_720P: return "720"; + case NEURD_INPUT_AUTOSCALING_1080P: return "1080"; + case NEURD_INPUT_AUTOSCALING_1440P: return "1440"; + default: return "?"; + } +} + +const char * +backend_choice_str(int b) +{ + return b == BACKEND_AUTO ? "auto" : backend_str((enum NeurD_backend)b); +} + +struct knobs +read_knobs() +{ + struct knobs k = {}; + k.enabled = true; + k.backend = BACKEND_DIRECTML; + k.autoscaling = NEURD_INPUT_AUTOSCALING_720P; + k.view_gain = 1.0f; + k.conv_gain = 0.4f; + k.interactive_min = 0; + k.video_model = NEURD_MODEL_VIDEO_RELATIVE_FAST; + k.depth_gain = kDefaultDepthGain; + k.dilate = kDefaultDilate; + + const char *e = std::getenv("DXR_LEIA_LIFT"); + if (e != nullptr && (e[0] == '0' || env_ieq(e, "off") || env_ieq(e, "false"))) { + k.enabled = false; + } + + // Explicit 64-bit view: the service is 64-bit, but a WOW64 host must not + // be redirected to Wow6432Node. + HKEY reg = nullptr; + if (RegOpenKeyExW(HKEY_LOCAL_MACHINE, kLiftRegKey, 0, KEY_QUERY_VALUE | KEY_WOW64_64KEY, ®) != ERROR_SUCCESS) { + reg = nullptr; + } + + char v[256]; + knob_src src; + + src = knob_lookup(reg, "DXR_LEIA_LIFT_BACKEND", L"Backend", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_backend(v, &k.backend)) { + k.src_backend = src; + } else { + U_LOG_W("Leia lift: backend '%s' (%s) not recognised (auto|directml|cuda|openvino) — using directml", v, + knob_src_str(src)); + } + } + + src = knob_lookup(reg, "DXR_LEIA_LIFT_SCALE", L"Scale", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + k.scale_forced = true; // as before: a set-but-unrecognised value still forces 720 + if (parse_scale(v, &k.autoscaling)) { + k.src_scale = src; + } else { + U_LOG_W("Leia lift: scale '%s' (%s) not recognised (720|1080|1440|none) — using 720", v, + knob_src_str(src)); + } + } + + src = knob_lookup(reg, "DXR_LEIA_LIFT_VIEW_GAIN", L"ViewGain", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_float_in(v, 0.0f, 10.0f, &k.view_gain)) { + k.src_view_gain = src; + } else { + U_LOG_W("Leia lift: view gain '%s' (%s) not a number in [0,10] — using 1.0", v, knob_src_str(src)); + } + } + + src = knob_lookup(reg, "DXR_LEIA_LIFT_CONV_GAIN", L"ConvGain", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_float_in(v, -2.0f, 2.0f, &k.conv_gain)) { + k.src_conv_gain = src; + } else { + U_LOG_W("Leia lift: conv gain '%s' (%s) not a number in [-2,2] — using 0.4", v, knob_src_str(src)); + } + } + + src = knob_lookup(reg, "DXR_LEIA_LIFT_INTERACTIVE_MIN", L"InteractiveMin", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_interactive_min(v, &k.interactive_min)) { + k.src_interactive_min = src; + } else { + U_LOG_W("Leia lift: interactive min '%s' (%s) not a version (e.g. 0.4.4) — ignored", v, + knob_src_str(src)); + } + } + + // The video streams' depth model (NeurD_init_with_options, 0.4.6+): 'fast' = the relative + // real-time model, 'metric' = NEURD_MODEL_VIDEO_METRIC_QUALITY (metric depth, closer to what the + // web path's MoGe gives). Photo streams always use the relative photo model (the only one the + // photo slot accepts). + src = knob_lookup(reg, "DXR_LEIA_LIFT_DEPTH_GAIN", L"DepthGain", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_float_in(v, 0.0f, 10.0f, &k.depth_gain)) { + k.src_depth_gain = src; + } else { + U_LOG_W("Leia lift: depth gain '%s' (%s) not a number in [0,10] — using %.1f", v, knob_src_str(src), + (double)kDefaultDepthGain); + } + } + + src = knob_lookup(reg, "DXR_LEIA_LIFT_DILATE", L"Dilate", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + float d = 0.0f; + if (parse_float_in(v, 0.0f, 16.0f, &d)) { + k.dilate = (int32_t)(d + 0.5f); + k.src_dilate = src; + } else { + U_LOG_W("Leia lift: dilate '%s' (%s) not a number in [0,16] — using %d", v, knob_src_str(src), + (int)kDefaultDilate); + } + } + + src = knob_lookup(reg, "DXR_LEIA_LIFT_VIDEO_MODEL", L"VideoModel", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (env_ieq(v, "metric")) { + k.video_model = NEURD_MODEL_VIDEO_METRIC_QUALITY; + k.src_video_model = src; + } else if (env_ieq(v, "fast")) { + k.video_model = NEURD_MODEL_VIDEO_RELATIVE_FAST; + k.src_video_model = src; + } else { + U_LOG_W("Leia lift: video model '%s' (%s) not recognised (fast|metric) — using fast", v, + knob_src_str(src)); + } + } + + if (reg != nullptr) { + RegCloseKey(reg); + } + return k; +} + +/* + * + * Compute shaders for the bridge (compiled at runtime on NeurD's device). + * + */ + +// Bridge texture (typed RGBA8/BGRA8 UNORM view — the view's format swizzles, so +// .rgba is always R,G,B,A) -> tightly packed RGBA8 raw buffer. +const char *kPackCs = R"( +cbuffer P : register(b0) { uint W; uint H; uint StrideBytes; uint Pad; }; +Texture2D Src : register(t0); +RWByteAddressBuffer Dst : register(u0); +[numthreads(16, 16, 1)] +void main(uint3 id : SV_DispatchThreadID) +{ + if (id.x >= W || id.y >= H) return; + uint4 u = (uint4)round(saturate(Src.Load(int3(id.xy, 0))) * 255.0); + Dst.Store(id.y * StrideBytes + id.x * 4, u.r | (u.g << 8) | (u.b << 16) | (u.a << 24)); +} +)"; + +// NeurD's RGBA8 raw output buffer -> bridge output texture. +const char *kUnpackCs = R"( +cbuffer P : register(b0) { uint W; uint H; uint StrideBytes; uint Pad; }; +RWByteAddressBuffer Src : register(u0); +RWTexture2D Dst : register(u1); +[numthreads(16, 16, 1)] +void main(uint3 id : SV_DispatchThreadID) +{ + if (id.x >= W || id.y >= H) return; + uint v = Src.Load(id.y * StrideBytes + id.x * 4); + Dst[id.xy] = float4(v & 0xFF, (v >> 8) & 0xFF, (v >> 16) & 0xFF, (v >> 24) & 0xFF) / 255.0; +} +)"; + +// NeurD's RGBA8 depth buffer -> single-channel R8 bridge (DEPTH contract: one +// channel). NeurD writes the normalised disparity replicated into RGB; R is kept. +// Polarity: NeurD is disparity-like (near = HIGH); the lift spec's RELATIVE is +// larger = FARTHER, so the value is flipped here. Dst is an R8_UNORM UAV, so +// 1 - v stays in [0,1]. +const char *kUnpackR8Cs = R"( +cbuffer P : register(b0) { uint W; uint H; uint StrideBytes; uint Pad; }; +RWByteAddressBuffer Src : register(u0); +RWTexture2D Dst : register(u1); +[numthreads(16, 16, 1)] +void main(uint3 id : SV_DispatchThreadID) +{ + if (id.x >= W || id.y >= H) return; + Dst[id.xy] = 1.0 - (float)(Src.Load(id.y * StrideBytes + id.x * 4) & 0xFF) / 255.0; +} +)"; + +/* + * + * Process-wide NeurD instance. + * + */ + +enum gstate : uint32_t +{ + G_UNPROBED = 0, //!< Nothing looked at yet. + G_ABSENT, //!< No NeurD.dll found — permanent for the process. + G_LOADING, //!< Background worker is loading / initialising. + G_RETRY_WAIT, //!< Licence activation hit the network; retry after next_retry_ms. + G_READY, //!< Up; converts allowed. + G_FAILED, //!< Permanent failure (bad licence, no DX device, ABI...). +}; + +struct global +{ + std::mutex mtx; //!< Serialises every NeurD call + NeurD-context use + the fields below. + + std::atomic state{G_UNPROBED}; + std::atomic worker_running{false}; + std::atomic next_retry_ms{0}; + std::atomic latency_ns{0}; //!< EMA of full convert wall time; 0 = none yet. + std::atomic streams_live{0}; + + int requested_backend = BACKEND_DIRECTML; //!< First acquirer's choice (NeurD's forced backend is sticky). + int32_t default_autoscaling = NEURD_INPUT_AUTOSCALING_720P; + struct knobs k0 = {}; //!< First acquirer's knobs (logged + interactive_min at activation). + + HMODULE lib = nullptr; + struct NeurD const *nd = nullptr; + enum NeurD_backend backend = NEURD_BACKEND_DIRECTML; + char backend_name[32] = {}; + + ID3D11Device *nd_dev = nullptr; //!< NeurD-owned (never Released) unless nd_dev_owned. + bool nd_dev_owned = false; //!< nd_dev is the plug-in's own device (NeurD < 0.4.3): Release it. + ID3D11DeviceContext *nd_ctx = nullptr; + ID3D11Query *nd_done = nullptr; + ID3D11ComputeShader *cs_pack = nullptr; + ID3D11ComputeShader *cs_unpack = nullptr; + ID3D11ComputeShader *cs_unpack_r8 = nullptr; + ID3D11Buffer *cb = nullptr; + LUID nd_luid = {}; + + int refcount = 0; + uint64_t next_stream_id = 1; + //! Written under g.mtx; atomic because caps reads it lock-free. + std::atomic interactive_unavailable{false}; + //! DXR_LEIA_LIFT_INTERACTIVE_MIN admitted a NeurD older than the header's + //! INTRODUCED_IN: call the table slot directly (the inline wrapper would + //! re-check 0.4.5 and return OUTDATED_RUNTIME). Set at activation. + bool interactive_direct_slot = false; + + //! Last-applied global NeurD properties (avoid a worker round-trip per prop per frame). + bool props_valid = false; + int32_t p_out_type = -1, p_tiles_w = -1, p_tiles_h = -1, p_inpaint = -1, p_autoscale = -1, p_autoconv = -1; + float p_conv = -1.0f, p_gain = -1.0f; + int32_t p_dilate = -1; +}; + +global g; + +uint64_t +now_ms() +{ + return GetTickCount64(); +} + +/* + * DLL location — mirrors NeurD's own loader order. Presence probe is separate + * from the load so caps can say "activating" without pulling NeurD (and its + * CUDA/DirectML/OpenVINO dependency tree) into the process on the caller's + * thread. + */ +bool +registry_neurd_path(char *out, size_t cap) +{ + HKEY key = nullptr; + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "SOFTWARE\\LeiaInc\\NeurD", 0, KEY_READ, &key) != ERROR_SUCCESS) { + return false; + } + char dir[MAX_PATH] = {}; + DWORD sz = sizeof(dir); + LSTATUS r = RegGetValueA(key, nullptr, nullptr, RRF_RT_REG_SZ, nullptr, dir, &sz); + RegCloseKey(key); + if (r != ERROR_SUCCESS || dir[0] == '\0') { + return false; + } + size_t n = strlen(dir); + const char *sep = (dir[n - 1] == '\\' || dir[n - 1] == '/') ? "" : "\\"; + int w = snprintf(out, cap, "%s%sNeurD.dll", dir, sep); + return w > 0 && (size_t)w < cap; +} + +bool +file_exists(const char *path) +{ + DWORD a = GetFileAttributesA(path); + return a != INVALID_FILE_ATTRIBUTES && (a & FILE_ATTRIBUTE_DIRECTORY) == 0; +} + +bool +neurd_present() +{ + char buf[MAX_PATH]; + if (SearchPathA(nullptr, "NeurD.dll", nullptr, MAX_PATH, buf, nullptr) != 0) { + return true; + } + const char *np = std::getenv("NEURD_PATH"); + if (np != nullptr && np[0] != '\0' && file_exists(np)) { + return true; + } + return registry_neurd_path(buf, sizeof(buf)) && file_exists(buf); +} + +HMODULE +neurd_load_library(const char **out_where) +{ + HMODULE h = LoadLibraryA("NeurD.dll"); + if (h != nullptr) { + *out_where = "PATH"; + return h; + } + const char *np = std::getenv("NEURD_PATH"); + if (np != nullptr && np[0] != '\0') { + h = LoadLibraryExA(np, nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); + if (h != nullptr) { + *out_where = "NEURD_PATH"; + return h; + } + } + char path[MAX_PATH]; + if (registry_neurd_path(path, sizeof(path))) { + h = LoadLibraryExA(path, nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); + if (h != nullptr) { + *out_where = "HKLM\\SOFTWARE\\LeiaInc\\NeurD"; + return h; + } + } + return nullptr; +} + +void +neurd_log_cb(const char *msg, int size) +{ + if (msg == nullptr || size <= 0) { + return; + } + // Not NUL-terminated (NeurD contract). Trim a trailing newline. + while (size > 0 && (msg[size - 1] == '\n' || msg[size - 1] == '\r')) { + size--; + } + // Until the module is READY these are one-off activation/init lines (licence, model paths, + // adapter) and the only place NeurD says WHY an init failed — surface them at the default + // log level. Once READY they can be per-stream chatter, so drop back to INFO. + if (g.state.load() != G_READY) { + U_LOG_W("NeurD: %.*s", size, msg); + } else { + U_LOG_I("NeurD: %.*s", size, msg); + } +} + +bool +enable_mt_protection(ID3D11DeviceContext *ctx, const char *who) +{ + ID3D11Multithread *mt = nullptr; + if (FAILED(ctx->QueryInterface(__uuidof(ID3D11Multithread), (void **)&mt)) || mt == nullptr) { + U_LOG_W("Leia lift: %s context has no ID3D11Multithread — concurrent use is unsafe", who); + return false; + } + if (!mt->GetMultithreadProtected()) { + mt->SetMultithreadProtected(TRUE); + U_LOG_W("Leia lift: enabled ID3D11Multithread protection on the %s immediate context", who); + } + mt->Release(); + return true; +} + +bool +device_luid(ID3D11Device *dev, LUID *out) +{ + IDXGIDevice *dxgi = nullptr; + if (FAILED(dev->QueryInterface(__uuidof(IDXGIDevice), (void **)&dxgi)) || dxgi == nullptr) { + return false; + } + IDXGIAdapter *ad = nullptr; + bool ok = false; + if (SUCCEEDED(dxgi->GetAdapter(&ad)) && ad != nullptr) { + DXGI_ADAPTER_DESC d = {}; + if (SUCCEEDED(ad->GetDesc(&d))) { + *out = d.AdapterLuid; + ok = true; + } + ad->Release(); + } + dxgi->Release(); + return ok; +} + +ID3D11ComputeShader * +compile_cs(ID3D11Device *dev, const char *src, const char *name) +{ + ID3DBlob *blob = nullptr; + ID3DBlob *err = nullptr; + HRESULT hr = D3DCompile(src, strlen(src), name, nullptr, nullptr, "main", "cs_5_0", 0, 0, &blob, &err); + if (FAILED(hr)) { + U_LOG_E("Leia lift: %s compile failed: %s", name, + err != nullptr ? (const char *)err->GetBufferPointer() : "(no log)"); + safe_release(err); + return nullptr; + } + safe_release(err); + ID3D11ComputeShader *cs = nullptr; + hr = dev->CreateComputeShader(blob->GetBufferPointer(), blob->GetBufferSize(), nullptr, &cs); + blob->Release(); + if (FAILED(hr)) { + U_LOG_E("Leia lift: CreateComputeShader(%s) failed: 0x%08x", name, (unsigned)hr); + return nullptr; + } + return cs; +} + +/*! + * NeurD_convert_stream_dx_interactive, honouring DXR_LEIA_LIFT_INTERACTIVE_MIN: + * the header's inline wrapper re-checks version >= INTRODUCED_IN (0.4.5), so + * when the override admitted an older-reporting runtime the table slot is + * called directly (same arguments minus @p nd). Called with g.mtx held. + */ +enum NeurD_status +nd_convert_stream_dx_interactive(struct NeurD const *nd, struct NeurD_stream *stream, struct NeurD_image const *input, + const float *viewpoints_xyz, int size, enum NeurD_pixel_format output_pix_fmt, + struct NeurD_image *output) +{ + if (g.interactive_direct_slot && nd->convert_stream_dx_interactive != nullptr) { + return nd->convert_stream_dx_interactive(stream, input, viewpoints_xyz, size, output_pix_fmt, output); + } + return NeurD_convert_stream_dx_interactive(nd, stream, input, viewpoints_xyz, size, output_pix_fmt, output); +} + +//! Drop g.nd_dev: Released only when the plug-in created it. Called with g.mtx held. +void +release_nd_dev_locked() +{ + if (g.nd_dev_owned) { + safe_release(g.nd_dev); + } + g.nd_dev = nullptr; + g.nd_dev_owned = false; +} + +//! Build the NeurD-device side of the bridge. Called with g.mtx held. +bool +setup_nd_device_locked() +{ + if (LEIA_NEURD_HAS(g.nd, get_dx_device)) { + g.nd_dev = static_cast(NeurD_get_dx_device(g.nd)); + g.nd_dev_owned = false; + if (g.nd_dev == nullptr) { + U_LOG_W("Leia lift: NeurD backend '%s' exposes no D3D11 device — the D3D11 lift path needs the " + "DirectML backend (DXR_LEIA_LIFT_BACKEND=directml). Lift unavailable.", + g.backend_name); + return false; + } + } else { + // NeurD < 0.4.3: no get_dx_device. NeurD takes the device from the + // input buffer (GetDevice) and returns its output on that device, so + // the plug-in supplies the device — default adapter, as NeurD's own + // example does. + ID3D11Device *dev = nullptr; + ID3D11DeviceContext *ctx = nullptr; + HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, + nullptr, 0, D3D11_SDK_VERSION, &dev, nullptr, &ctx); + if (FAILED(hr) || dev == nullptr || ctx == nullptr) { + U_LOG_W("Leia lift: NeurD < 0.4.3 and D3D11CreateDevice for the plug-in's bridge device failed " + "(0x%08x) — lift unavailable", + (unsigned)hr); + safe_release(ctx); + safe_release(dev); + return false; + } + // GetImmediateContext below takes its own reference. + ctx->Release(); + g.nd_dev = dev; + g.nd_dev_owned = true; + LUID l = {}; + const bool have_luid = device_luid(dev, &l); + U_LOG_W("Leia lift: NeurD < 0.4.3: no get_dx_device — using the plug-in's own D3D11 device on the " + "default adapter (LUID %08lx:%08lx)%s", + have_luid ? (unsigned long)l.HighPart : 0UL, have_luid ? (unsigned long)l.LowPart : 0UL, + g.backend == NEURD_BACKEND_CUDA ? "" + : " — NeurD 0.3.x's DX path needs the CUDA backend " + "(DXR_LEIA_LIFT_BACKEND=cuda); converts may fail"); + } + g.nd_dev->GetImmediateContext(&g.nd_ctx); + enable_mt_protection(g.nd_ctx, "NeurD"); + + D3D11_QUERY_DESC qd = {D3D11_QUERY_EVENT, 0}; + g.cs_pack = compile_cs(g.nd_dev, kPackCs, "leia_lift_pack"); + g.cs_unpack = compile_cs(g.nd_dev, kUnpackCs, "leia_lift_unpack"); + g.cs_unpack_r8 = compile_cs(g.nd_dev, kUnpackR8Cs, "leia_lift_unpack_r8"); + D3D11_BUFFER_DESC bd = {}; + bd.ByteWidth = 16; + bd.Usage = D3D11_USAGE_DYNAMIC; + bd.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + bd.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + if (g.cs_pack == nullptr || g.cs_unpack == nullptr || g.cs_unpack_r8 == nullptr || FAILED(g.nd_dev->CreateQuery(&qd, &g.nd_done)) || + FAILED(g.nd_dev->CreateBuffer(&bd, nullptr, &g.cb))) { + U_LOG_W("Leia lift: bridge setup on NeurD's device failed — lift unavailable"); + safe_release(g.cs_pack); + safe_release(g.cs_unpack); + safe_release(g.cs_unpack_r8); + safe_release(g.nd_done); + safe_release(g.cb); + safe_release(g.nd_ctx); + release_nd_dev_locked(); + return false; + } + if (device_luid(g.nd_dev, &g.nd_luid)) { + U_LOG_W("Leia lift: NeurD D3D11 device on adapter LUID %08lx:%08lx (FL 0x%x)", + (unsigned long)g.nd_luid.HighPart, (unsigned long)g.nd_luid.LowPart, + (unsigned)g.nd_dev->GetFeatureLevel()); + } + return true; +} + +/*! + * Background worker: load (once) + init (retryable). Runs detached; publishes + * the outcome through g.state. + */ +void +activation_worker() +{ + uint32_t next = G_FAILED; + { + // Load happens once per process. NOT under g.mtx for the slow part — + // but nothing else touches g.lib / g.nd until state says READY. + if (g.nd == nullptr) { + const char *where = "?"; + HMODULE lib = neurd_load_library(&where); + if (lib == nullptr) { + U_LOG_W("Leia lift: NeurD.dll present but LoadLibrary failed (err %lu) — lift unavailable", + (unsigned long)GetLastError()); + g.state.store(G_FAILED); + g.worker_running.store(false); + return; + } + auto load = reinterpret_cast(reinterpret_cast(GetProcAddress(lib, "NeurD_load"))); + struct NeurD_load_request req = {NEURD_VERSION}; + struct NeurD const *nd = load != nullptr ? load(&req) : nullptr; + if (nd == nullptr) { + U_LOG_W("Leia lift: NeurD.dll (%s) has no usable NeurD_load — lift unavailable", where); + // Deliberately no FreeLibrary: NeurD may have spun threads. + g.state.store(G_FAILED); + g.worker_running.store(false); + return; + } + U_LOG_W("Leia lift: loaded NeurD %u.%u.%u from %s (plug-in built against %u.%u.%u)", + NEURD_GET_VERSION_MAJOR(nd->version), NEURD_GET_VERSION_MINOR(nd->version), + NEURD_GET_VERSION_PATCH(nd->version), where, + NEURD_GET_VERSION_MAJOR(NEURD_VERSION), + NEURD_GET_VERSION_MINOR(NEURD_VERSION), + NEURD_GET_VERSION_PATCH(NEURD_VERSION)); + if (NEURD_GET_VERSION_MAJOR(nd->version) != 0 || + !LEIA_NEURD_HAS(nd, convert_stream_dx) || !LEIA_NEURD_HAS(nd, create_stream) || + !LEIA_NEURD_HAS(nd, set_prop_1i) || !LEIA_NEURD_HAS(nd, set_prop_1f)) { + U_LOG_W("Leia lift: NeurD runtime too old/new for the D3D11 stream path (need 0.3.11+, " + "major 0) — lift unavailable"); + g.state.store(G_FAILED); + g.worker_running.store(false); + return; + } + g.lib = lib; + g.nd = nd; + if (LEIA_NEURD_HAS(nd, set_logger_callback)) { + NeurD_set_logger_callback(nd, neurd_log_cb); + } + } + + struct NeurD const *nd = g.nd; + + // Properties that must precede init (NeurD reads some at init time): + // the defaults this plug-in wants for every stream. + NeurD_set_prop_1i(nd, NEURD_PROP_OUTPUT_TYPE, NEURD_OUTPUT_IMAGE_TYPE_SBS); + NeurD_set_prop_1i(nd, NEURD_PROP_INPUT_AUTOSCALING, g.default_autoscaling); + NeurD_set_prop_1i(nd, NEURD_PROP_INPAINT_TYPE, NEURD_INPAINT_TYPE_V1_STRETCH); + + // Backend must be forced immediately before init. NeurD keeps a forced + // backend across deinit, so this is effectively set once per process. + if (g.requested_backend != BACKEND_AUTO && LEIA_NEURD_HAS(nd, set_backend)) { + enum NeurD_status bs = NeurD_set_backend(nd, (enum NeurD_backend)g.requested_backend); + if (bs != NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD_set_backend(%s) -> %s", backend_str((enum NeurD_backend)g.requested_backend), + status_str(bs)); + } + } + + enum NeurD_status st; + if (LEIA_NEURD_HAS(nd, init_with_options)) { + struct NeurD_init_options opts = {}; + opts.struct_size = sizeof(opts); + opts.photo_model_id = NEURD_MODEL_PHOTO_RELATIVE_QUALITY; + opts.video_model_id = g.k0.video_model; + st = NeurD_init_with_options(nd, 1 /* multithreaded: we call from the runtime's lift thread */, &opts); + } else if (LEIA_NEURD_HAS(nd, init)) { + st = NeurD_init(nd, 1); + } else { + st = NEURD_UNAVAILABLE_OUTDATED_RUNTIME; + } + + if (st == NEURD_LICENSE_NETWORK_ERROR || st == NEURD_NETWORK_ERROR) { + g.next_retry_ms.store(now_ms() + kActivationRetryMs); + U_LOG_W("Leia lift: NeurD licence activation needs the network (%s) — reporting ACTIVATING, " + "retrying in %llus", + status_str(st), (unsigned long long)(kActivationRetryMs / 1000)); + g.state.store(G_RETRY_WAIT); + g.worker_running.store(false); + return; + } + if (st != NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD init failed: %s — lift unavailable", status_str(st)); + g.state.store(G_FAILED); + g.worker_running.store(false); + return; + } + + enum NeurD_backend be = (g.requested_backend == BACKEND_AUTO) + ? NEURD_BACKEND_DIRECTML + : (enum NeurD_backend)g.requested_backend; + if (LEIA_NEURD_HAS(nd, get_backend)) { + NeurD_get_backend(nd, &be); + } + + std::lock_guard lock(g.mtx); + g.backend = be; + snprintf(g.backend_name, sizeof(g.backend_name), "neurd-%s", backend_str(be)); + const struct knobs &k0 = g.k0; + uint64_t interactive_min = NEURD_convert_stream_dx_interactive_INTRODUCED_IN; + if (k0.interactive_min != 0) { + interactive_min = k0.interactive_min; + } + U_LOG_W("Leia lift: knobs backend=%s(%s) interactive_min=%u.%u.%u(%s) scale=%s(%s) view_gain=%.2f(%s) " + "conv_gain=%.2f(%s) video_model=%s(%s) depth_gain=%.2f(%s) dilate=%d(%s) [env > HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift > default]", + backend_choice_str(k0.backend), knob_src_str(k0.src_backend), + (unsigned)NEURD_GET_VERSION_MAJOR(interactive_min), (unsigned)NEURD_GET_VERSION_MINOR(interactive_min), + (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), knob_src_str(k0.src_interactive_min), + k0.scale_forced ? scale_str(k0.autoscaling) : "per-stream", knob_src_str(k0.src_scale), + (double)k0.view_gain, knob_src_str(k0.src_view_gain), (double)k0.conv_gain, + knob_src_str(k0.src_conv_gain), + k0.video_model == NEURD_MODEL_VIDEO_METRIC_QUALITY ? "metric" : "fast", + knob_src_str(k0.src_video_model), (double)k0.depth_gain, knob_src_str(k0.src_depth_gain), (int)k0.dilate, + knob_src_str(k0.src_dilate)); + if (k0.interactive_min != 0) { + U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN=%u.%u.%u (%s) — assuming interactive convert on NeurD " + "%u.%u.%u (only for the internal-interactive dev package; a stock 0.4.4 will crash here)", + (unsigned)NEURD_GET_VERSION_MAJOR(interactive_min), + (unsigned)NEURD_GET_VERSION_MINOR(interactive_min), + (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), knob_src_str(k0.src_interactive_min), + (unsigned)NEURD_GET_VERSION_MAJOR(nd->version), (unsigned)NEURD_GET_VERSION_MINOR(nd->version), + (unsigned)NEURD_GET_VERSION_PATCH(nd->version)); + } + g.interactive_unavailable = !(nd->version >= interactive_min && nd->convert_stream_dx_interactive != nullptr); + g.interactive_direct_slot = !g.interactive_unavailable.load() && + nd->version < NEURD_convert_stream_dx_interactive_INTRODUCED_IN; + g.props_valid = false; + next = setup_nd_device_locked() ? (uint32_t)G_READY : (uint32_t)G_FAILED; + if (next == G_READY) { + U_LOG_W("Leia lift: NeurD READY — backend %s, interactive viewpoints %s", g.backend_name, + g.interactive_unavailable ? "UNAVAILABLE (NeurD < 0.4.5)" : "available"); + } + } + g.state.store(next); + g.worker_running.store(false); +} + +/*! + * Advance the process state machine without blocking: probe presence on first + * touch, and (re)start the activation worker when due. + */ +void +kick(const struct knobs &k) +{ + uint32_t s = g.state.load(); + if (s == G_UNPROBED) { + // Racing first callers are fine: presence is idempotent and only one + // wins the worker_running exchange below. + if (!neurd_present()) { + uint32_t expect = G_UNPROBED; + if (g.state.compare_exchange_strong(expect, G_ABSENT)) { + U_LOG_W("Leia lift: NeurD.dll not found (PATH / NEURD_PATH / HKLM\\SOFTWARE\\LeiaInc\\NeurD) — " + "2D->3D lift unavailable; plug-in otherwise unaffected"); + } + return; + } + } else if (s == G_RETRY_WAIT) { + if (now_ms() < g.next_retry_ms.load()) { + return; + } + } else { + return; // ABSENT / LOADING / READY / FAILED: nothing to do. + } + + bool expect_idle = false; + if (!g.worker_running.compare_exchange_strong(expect_idle, true)) { + return; + } + if (s == G_UNPROBED) { + g.requested_backend = k.backend; + g.default_autoscaling = k.autoscaling; + g.k0 = k; + } + g.state.store(G_LOADING); + try { + std::thread(activation_worker).detach(); + } catch (...) { + U_LOG_W("Leia lift: could not start the NeurD activation thread"); + g.state.store(s); + g.worker_running.store(false); + } +} + +uint32_t +public_state(uint32_t s) +{ + switch (s) { + case G_READY: return LEIA_LIFT_STATE_READY; + case G_UNPROBED: + case G_LOADING: + case G_RETRY_WAIT: return LEIA_LIFT_STATE_ACTIVATING; + default: return LEIA_LIFT_STATE_UNAVAILABLE; + } +} + +/* + * + * Per-stream bridge. + * + */ + +struct lift_stream +{ + uint64_t id; + uint32_t mode; + uint32_t content_hint; + float input_scale; + struct NeurD_stream *ns; //!< Created lazily on first convert (NeurD must be READY). + + ID3D11Device *dev; //!< Caller's device (AddRef'd) the bridges are opened on. + ID3D11Query *our_done; + + // Input bridge. + uint32_t in_w, in_h; + DXGI_FORMAT in_family; //!< R8G8B8A8_TYPELESS or B8G8R8A8_TYPELESS. + ID3D11Texture2D *in_nd; + ID3D11Texture2D *in_ours; + ID3D11ShaderResourceView *in_srv; + ID3D11Buffer *in_buf; + ID3D11UnorderedAccessView *in_uav; + + // Output bridge. + uint32_t out_w, out_h; + DXGI_FORMAT out_fmt; //!< R8G8B8A8_UNORM, or R8_UNORM for DEPTH + ID3D11Texture2D *out_nd; + ID3D11Texture2D *out_ours; + ID3D11UnorderedAccessView *out_uav; + ID3D11Resource *out_src_key; //!< NeurD output resource the raw UAV below was made for. + ID3D11UnorderedAccessView *out_src_uav; +}; + +void +release_in_bridge(lift_stream *s) +{ + safe_release(s->in_uav); + safe_release(s->in_buf); + safe_release(s->in_srv); + safe_release(s->in_ours); + safe_release(s->in_nd); + s->in_w = s->in_h = 0; +} + +void +release_out_bridge(lift_stream *s) +{ + safe_release(s->out_src_uav); + s->out_src_key = nullptr; // not owned + safe_release(s->out_uav); + safe_release(s->out_ours); + safe_release(s->out_nd); + s->out_w = s->out_h = 0; + s->out_fmt = DXGI_FORMAT_UNKNOWN; +} + +//! Called with g.mtx held. +void +release_stream_locked(lift_stream *s) +{ + release_in_bridge(s); + release_out_bridge(s); + safe_release(s->our_done); + safe_release(s->dev); + if (s->ns != nullptr && g.state.load() == G_READY && LEIA_NEURD_HAS(g.nd, destroy_stream)) { + NeurD_destroy_stream(g.nd, s->ns); + } + s->ns = nullptr; +} + +/*! + * Shared texture on NeurD's device, opened on the caller's device. Legacy + * (non-NT) handle — the one NeurD itself and LeiaMeet rely on; not owned, no + * CloseHandle. + */ +bool +make_shared_tex(ID3D11Device *ours, uint32_t w, uint32_t h, DXGI_FORMAT fmt, UINT bind, ID3D11Texture2D **out_nd, + ID3D11Texture2D **out_ours) +{ + D3D11_TEXTURE2D_DESC td = {}; + td.Width = w; + td.Height = h; + td.MipLevels = 1; + td.ArraySize = 1; + td.Format = fmt; + td.SampleDesc.Count = 1; + td.Usage = D3D11_USAGE_DEFAULT; + td.BindFlags = bind; + td.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + if (FAILED(g.nd_dev->CreateTexture2D(&td, nullptr, out_nd))) { + return false; + } + IDXGIResource *dr = nullptr; + HANDLE hshared = nullptr; + bool ok = SUCCEEDED((*out_nd)->QueryInterface(__uuidof(IDXGIResource), (void **)&dr)) && dr != nullptr && + SUCCEEDED(dr->GetSharedHandle(&hshared)) && hshared != nullptr && + SUCCEEDED(ours->OpenSharedResource(hshared, __uuidof(ID3D11Texture2D), (void **)out_ours)); + safe_release(dr); + if (!ok) { + safe_release(*out_nd); + return false; + } + return true; +} + +bool +ensure_in_bridge(lift_stream *s, uint32_t w, uint32_t h, DXGI_FORMAT family) +{ + if (s->in_nd != nullptr && s->in_w == w && s->in_h == h && s->in_family == family) { + return true; + } + release_in_bridge(s); + if (!make_shared_tex(s->dev, w, h, family, D3D11_BIND_SHADER_RESOURCE, &s->in_nd, &s->in_ours)) { + LIFT_WARN_ONCE("Leia lift: input bridge texture %ux%u create/open failed", w, h); + return false; + } + D3D11_SHADER_RESOURCE_VIEW_DESC sd = {}; + sd.Format = (family == DXGI_FORMAT_B8G8R8A8_TYPELESS) ? DXGI_FORMAT_B8G8R8A8_UNORM : DXGI_FORMAT_R8G8B8A8_UNORM; + sd.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + sd.Texture2D.MipLevels = 1; + + // Raw input buffer NeurD accepts: flat RGBA8, stride w*4, SHARED. + D3D11_BUFFER_DESC bd = {}; + bd.ByteWidth = w * h * 4; + bd.Usage = D3D11_USAGE_DEFAULT; + bd.BindFlags = D3D11_BIND_UNORDERED_ACCESS | D3D11_BIND_SHADER_RESOURCE; + bd.MiscFlags = D3D11_RESOURCE_MISC_BUFFER_ALLOW_RAW_VIEWS | D3D11_RESOURCE_MISC_SHARED; + D3D11_UNORDERED_ACCESS_VIEW_DESC ud = {}; + ud.Format = DXGI_FORMAT_R32_TYPELESS; + ud.ViewDimension = D3D11_UAV_DIMENSION_BUFFER; + ud.Buffer.NumElements = w * h; + ud.Buffer.Flags = D3D11_BUFFER_UAV_FLAG_RAW; + + if (FAILED(g.nd_dev->CreateShaderResourceView(s->in_nd, &sd, &s->in_srv)) || + FAILED(g.nd_dev->CreateBuffer(&bd, nullptr, &s->in_buf)) || + FAILED(g.nd_dev->CreateUnorderedAccessView(s->in_buf, &ud, &s->in_uav))) { + LIFT_WARN_ONCE("Leia lift: input bridge views/buffer %ux%u failed", w, h); + release_in_bridge(s); + return false; + } + s->in_w = w; + s->in_h = h; + s->in_family = family; + return true; +} + +bool +ensure_out_bridge(lift_stream *s, uint32_t w, uint32_t h, DXGI_FORMAT fmt) +{ + if (s->out_nd != nullptr && s->out_w == w && s->out_h == h && s->out_fmt == fmt) { + return true; + } + release_out_bridge(s); + if (w > kMaxTexDim || h > kMaxTexDim) { + LIFT_WARN_ONCE("Leia lift: output %ux%u exceeds the D3D11 texture limit (%u) — lower view_count or " + "DXR_LEIA_LIFT_SCALE", + w, h, kMaxTexDim); + return false; + } + if (!make_shared_tex(s->dev, w, h, fmt, + D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_UNORDERED_ACCESS, &s->out_nd, &s->out_ours) || + FAILED(g.nd_dev->CreateUnorderedAccessView(s->out_nd, nullptr, &s->out_uav))) { + LIFT_WARN_ONCE("Leia lift: output bridge texture %ux%u failed", w, h); + release_out_bridge(s); + return false; + } + s->out_w = w; + s->out_h = h; + s->out_fmt = fmt; + return true; +} + +//! Spin until @p q (an EVENT query already End()ed on @p ctx) signals. +bool +drain(ID3D11DeviceContext *ctx, ID3D11Query *q, const char *who) +{ + ctx->End(q); + ctx->Flush(); + const uint64_t t0 = now_ms(); + HRESULT hr; + while ((hr = ctx->GetData(q, nullptr, 0, 0)) == S_FALSE) { + if (now_ms() - t0 > kDrainTimeoutMs) { + U_LOG_W("Leia lift: %s queue drain timed out (%llu ms) — device hung/lost?", who, + (unsigned long long)kDrainTimeoutMs); + return false; + } + std::this_thread::yield(); + } + if (hr != S_OK) { + U_LOG_W("Leia lift: %s queue drain failed 0x%08x — device lost?", who, (unsigned)hr); + return false; + } + return true; +} + +bool +set_cb(uint32_t a, uint32_t b, uint32_t c, uint32_t d) +{ + D3D11_MAPPED_SUBRESOURCE m = {}; + if (FAILED(g.nd_ctx->Map(g.cb, 0, D3D11_MAP_WRITE_DISCARD, 0, &m))) { + return false; + } + uint32_t *p = static_cast(m.pData); + p[0] = a; + p[1] = b; + p[2] = c; + p[3] = d; + g.nd_ctx->Unmap(g.cb, 0); + return true; +} + +void +unbind_cs() +{ + ID3D11UnorderedAccessView *nu[2] = {nullptr, nullptr}; + ID3D11ShaderResourceView *ns = nullptr; + g.nd_ctx->CSSetUnorderedAccessViews(0, 2, nu, nullptr); + g.nd_ctx->CSSetShaderResources(0, 1, &ns); + g.nd_ctx->CSSetShader(nullptr, nullptr, 0); +} + +inline UINT +groups16(uint32_t n) +{ + return (n + 15) / 16; +} + +//! Copy/unpack NeurD's output into the stream's output bridge (NeurD device). +bool +stage_output(lift_stream *s, const struct NeurD_image &out) +{ + auto *res = static_cast(out.data); + if (res == nullptr || out.width <= 0 || out.height <= 0) { + return false; + } + const uint32_t w = (uint32_t)out.width; + const uint32_t h = (uint32_t)out.height; + + ID3D11Texture2D *tex = nullptr; + if (SUCCEEDED(res->QueryInterface(__uuidof(ID3D11Texture2D), (void **)&tex)) && tex != nullptr) { + // NOTE: DEPTH on this path is NOT polarity-flipped (still NeurD's + // near = high) — only kUnpackR8Cs flips. Only NeurD 0.3.x's CUDA DX + // path returns a texture. + // A texture result is copied as-is (RGBA8 even for DEPTH; out_format + // says so). NeurD's stream path returns buffers; this is defensive. + if (!ensure_out_bridge(s, w, h, DXGI_FORMAT_R8G8B8A8_UNORM)) { + tex->Release(); + return false; + } + // Region copy: NeurD's internal texture may exceed the reported size. + D3D11_BOX box = {0, 0, 0, w, h, 1}; + g.nd_ctx->CopySubresourceRegion(s->out_nd, 0, 0, 0, 0, tex, 0, &box); + tex->Release(); + return true; + } + + ID3D11Buffer *buf = nullptr; + if (FAILED(res->QueryInterface(__uuidof(ID3D11Buffer), (void **)&buf)) || buf == nullptr) { + LIFT_WARN_ONCE("Leia lift: NeurD output is neither a texture nor a buffer"); + return false; + } + const bool depth = (s->mode == LEIA_LIFT_MODE_DEPTH); + if (!ensure_out_bridge(s, w, h, depth ? DXGI_FORMAT_R8_UNORM : DXGI_FORMAT_R8G8B8A8_UNORM)) { + buf->Release(); + return false; + } + D3D11_BUFFER_DESC bd = {}; + buf->GetDesc(&bd); + uint32_t stride = (out.stride >= (int32_t)(w * 4)) ? (uint32_t)out.stride : w * 4; + if ((uint64_t)stride * h > bd.ByteWidth) { + buf->Release(); + LIFT_WARN_ONCE("Leia lift: NeurD output buffer too small (%u < %llu bytes)", bd.ByteWidth, + (unsigned long long)stride * h); + return false; + } + if (s->out_src_key != res || s->out_src_uav == nullptr) { + safe_release(s->out_src_uav); + D3D11_UNORDERED_ACCESS_VIEW_DESC ud = {}; + ud.Format = DXGI_FORMAT_R32_TYPELESS; + ud.ViewDimension = D3D11_UAV_DIMENSION_BUFFER; + ud.Buffer.NumElements = bd.ByteWidth / 4; + ud.Buffer.Flags = D3D11_BUFFER_UAV_FLAG_RAW; + if (FAILED(g.nd_dev->CreateUnorderedAccessView(buf, &ud, &s->out_src_uav))) { + buf->Release(); + LIFT_WARN_ONCE("Leia lift: raw UAV over NeurD's output buffer failed"); + return false; + } + s->out_src_key = res; + } + buf->Release(); + + if (!set_cb(w, h, stride, 0)) { + return false; + } + ID3D11UnorderedAccessView *uavs[2] = {s->out_src_uav, s->out_uav}; + g.nd_ctx->CSSetShader(depth ? g.cs_unpack_r8 : g.cs_unpack, nullptr, 0); + g.nd_ctx->CSSetConstantBuffers(0, 1, &g.cb); + g.nd_ctx->CSSetUnorderedAccessViews(0, 2, uavs, nullptr); + g.nd_ctx->Dispatch(groups16(w), groups16(h), 1); + unbind_cs(); + return true; +} + +//! Apply one global NeurD property only if it changed. Called with g.mtx held. +bool +prop_i(enum NeurD_prop p, int32_t v, int32_t &cache) +{ + if (g.props_valid && cache == v) { + return true; + } + enum NeurD_status st = NeurD_set_prop_1i(g.nd, p, v); + if (st != NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD_set_prop_1i(%d, %d) -> %s", (int)p, (int)v, status_str(st)); + cache = -1; + return false; + } + cache = v; + return true; +} + +bool +prop_f(enum NeurD_prop p, float v, float &cache) +{ + if (g.props_valid && cache == v) { + return true; + } + enum NeurD_status st = NeurD_set_prop_1f(g.nd, p, v); + if (st != NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD_set_prop_1f(%d, %f) -> %s", (int)p, (double)v, status_str(st)); + cache = -1.0f; + return false; + } + cache = v; + return true; +} + + +int32_t +autoscale_for(const lift_stream *s, const struct knobs &k, uint32_t in_h) +{ + // An explicit DXR_LEIA_LIFT_SCALE is a developer override and wins; else + // the stream's input_scale (1 = native) picks the bucket; else the default. + if (k.scale_forced || !(s->input_scale > 0.0f) || s->input_scale > 1.0f) { + return k.autoscaling; + } + // Smallest NeurD bucket that still covers the requested inference height. + float target = s->input_scale * (float)in_h; + if (target <= 720.0f) { + return NEURD_INPUT_AUTOSCALING_720P; + } + if (target <= 1080.0f) { + return NEURD_INPUT_AUTOSCALING_1080P; + } + if (target <= 1440.0f) { + return NEURD_INPUT_AUTOSCALING_1440P; + } + return NEURD_INPUT_AUTOSCALING_NONE; +} + +} // namespace + +/* + * + * Per-DP handle. + * + */ + +struct leia_lift_neurd +{ + struct knobs k; + bool acquired; //!< Holds a ref on the process instance. + std::vector streams; +}; + +extern "C" void +leia_lift_neurd_map_viewpoint(const float in_m[3], float gain, float out_n[3]) +{ + out_n[0] = clampf(gain * in_m[0] / kIpdRefM, -kViewpointClamp, kViewpointClamp); + // A lenticular panel has horizontal parallax only. Passing the tracked eye + // HEIGHT made NeurD render the frame as seen from above/below the display + // centre: the lifted image shifted vertically with head height (visible + // "jumps" as the head moved) and a filled band appeared at the top edge + // (David, panel, 2026-09-26). Vertical viewpoint offset is never wanted. + (void)in_m[1]; + out_n[1] = 0.0f; + // NeurD's z is a dimensionless depth offset whose relation to viewer + // distance is not specified; head z is deliberately not mapped (see doc). + out_n[2] = 0.0f; +} + +extern "C" struct leia_lift_neurd * +leia_lift_neurd_create(void) +{ + auto *l = new (std::nothrow) leia_lift_neurd(); + if (l == nullptr) { + return nullptr; + } + l->k = read_knobs(); + l->acquired = false; + if (!l->k.enabled) { + LIFT_WARN_ONCE("Leia lift: disabled by DXR_LEIA_LIFT=0"); + } + return l; +} + +extern "C" void +leia_lift_neurd_destroy(struct leia_lift_neurd **plift) +{ + if (plift == nullptr || *plift == nullptr) { + return; + } + struct leia_lift_neurd *l = *plift; + *plift = nullptr; + { + std::lock_guard lock(g.mtx); + for (lift_stream *s : l->streams) { + release_stream_locked(s); + delete s; + g.streams_live.fetch_sub(1); + } + l->streams.clear(); + if (l->acquired) { + l->acquired = false; + if (--g.refcount == 0 && g.state.load() == G_READY && LEIA_NEURD_HAS(g.nd, shrink_memory_pool)) { + // Last user gone: give NeurD's pools back but stay initialised + // (re-init = licence + model load; DPs are recreated on focus change). + NeurD_shrink_memory_pool(g.nd); + g.props_valid = false; + } + } + } + delete l; +} + +extern "C" bool +leia_lift_neurd_get_caps(struct leia_lift_neurd *l, struct leia_lift_neurd_caps *out) +{ + if (out == nullptr) { + return false; + } + memset(out, 0, sizeof(*out)); + if (l == nullptr || !l->k.enabled) { + return true; // modes 0 / UNAVAILABLE — a valid answer, not a failure. + } + kick(l->k); + + uint32_t s = g.state.load(); + out->state = public_state(s); + if (out->state == LEIA_LIFT_STATE_UNAVAILABLE) { + return true; + } + // Never NVIEW: NeurD's Config::adjust() re-derives the tile grid from the + // output type on every convert (SBS 2x1, TB 1x2, DEPTH 1x1), and the public + // OUTPUT_TYPE prop accepts only 0..2, so OUTPUT_TILES_W/H are overridden — + // N > 2 tiles per convert is impossible through the public API (<= 0.4.6). + out->modes = LEIA_LIFT_MODE_DEPTH | LEIA_LIFT_MODE_SBS; + out->max_streams = kMaxStreams; + out->max_views = kMaxViews; + out->depth_semantics = 0; // relative: per-frame min-max normalised depth, larger = farther (NeurD's near=high disparity is flipped in kUnpackR8Cs) + uint64_t lat = g.latency_ns.load(); + if (s == G_READY) { + // No lock: backend/backend_name are written once, before the seq_cst + // store that publishes G_READY, and never again — caps must not wait + // behind a convert holding g.mtx. + snprintf(out->backend, sizeof(out->backend), "%s", g.backend_name); + if (lat == 0) { + lat = (g.backend == NEURD_BACKEND_DIRECTML) ? kPriorLatencyDirectMlNs + : (g.backend == NEURD_BACKEND_CUDA) ? kPriorLatencyCudaNs + : kPriorLatencyOtherNs; + } + } else { + snprintf(out->backend, sizeof(out->backend), "neurd-%s", + l->k.backend == BACKEND_AUTO ? "auto" : backend_str((enum NeurD_backend)l->k.backend)); + if (lat == 0) { + lat = kPriorLatencyDirectMlNs; + } + } + out->typical_latency_ns = lat; + return true; +} + +extern "C" bool +leia_lift_neurd_stream_create(struct leia_lift_neurd *l, const struct leia_lift_neurd_stream_desc *desc, uint64_t *out_id) +{ + if (l == nullptr || desc == nullptr || out_id == nullptr || !l->k.enabled) { + return false; + } + if (desc->mode == LEIA_LIFT_MODE_NVIEW) { + LIFT_WARN_ONCE("Leia lift: N-view not supported — NeurD public API tiles are fixed by output type; " + "N-view needs a tiled output type (ask Leia)"); + return false; + } + if (desc->mode != LEIA_LIFT_MODE_DEPTH && desc->mode != LEIA_LIFT_MODE_SBS) { + LIFT_WARN_ONCE("Leia lift: stream_create with unsupported mode %u", desc->mode); + return false; + } + kick(l->k); + if (public_state(g.state.load()) == LEIA_LIFT_STATE_UNAVAILABLE) { + return false; + } + + std::lock_guard lock(g.mtx); + if (g.streams_live.load() >= kMaxStreams) { + LIFT_WARN_ONCE("Leia lift: stream limit (%u, NeurD's MAX_STREAMS) reached", kMaxStreams); + return false; + } + auto *s = new (std::nothrow) lift_stream(); + if (s == nullptr) { + return false; + } + s->id = g.next_stream_id++; + s->mode = desc->mode; + s->content_hint = desc->content_hint; + s->input_scale = desc->input_scale; + l->streams.push_back(s); + g.streams_live.fetch_add(1); + if (!l->acquired) { + l->acquired = true; + g.refcount++; + } + *out_id = s->id; + U_LOG_W("Leia lift: stream %llu created (mode %u, %s)", (unsigned long long)s->id, s->mode, + s->content_hint == 1 ? "photo" : "video"); + return true; +} + +extern "C" void +leia_lift_neurd_stream_destroy(struct leia_lift_neurd *l, uint64_t id) +{ + if (l == nullptr) { + return; + } + std::lock_guard lock(g.mtx); + for (auto it = l->streams.begin(); it != l->streams.end(); ++it) { + if ((*it)->id == id) { + release_stream_locked(*it); + delete *it; + l->streams.erase(it); + g.streams_live.fetch_sub(1); + return; + } + } +} + +extern "C" bool +leia_lift_neurd_convert(struct leia_lift_neurd *l, + uint64_t id, + void *d3d11_context, + void *input, + uint32_t w, + uint32_t h, + const struct leia_lift_neurd_params *p, + const float *viewpoints_m, + uint32_t viewpoint_floats, + const float *eye_left, + const float *eye_right, + bool eyes_valid, + void **out_resource, + uint32_t *out_w, + uint32_t *out_h, + uint32_t *out_format) +{ + if (l == nullptr || !l->k.enabled || d3d11_context == nullptr || input == nullptr || w == 0 || h == 0 || + out_resource == nullptr || out_w == nullptr || out_h == nullptr || out_format == nullptr) { + return false; + } + *out_resource = nullptr; + + kick(l->k); + if (g.state.load() != G_READY) { + return false; // ACTIVATING or UNAVAILABLE — caps says which. + } + + LARGE_INTEGER qf, q0, q1; + QueryPerformanceFrequency(&qf); + QueryPerformanceCounter(&q0); + + try { + std::lock_guard lock(g.mtx); + struct NeurD const *nd = g.nd; + + lift_stream *s = nullptr; + for (lift_stream *it : l->streams) { + if (it->id == id) { + s = it; + break; + } + } + if (s == nullptr) { + LIFT_WARN_ONCE("Leia lift: convert on unknown stream id %llu", (unsigned long long)id); + return false; + } + + // ---- Validate the input. + auto *ctx = static_cast(d3d11_context); + auto *in_res = static_cast(input); + ID3D11Texture2D *in_tex = nullptr; + if (FAILED(in_res->QueryInterface(__uuidof(ID3D11Texture2D), (void **)&in_tex)) || in_tex == nullptr) { + LIFT_WARN_ONCE("Leia lift: input is not an ID3D11Texture2D"); + return false; + } + D3D11_TEXTURE2D_DESC idesc = {}; + in_tex->GetDesc(&idesc); + DXGI_FORMAT family = DXGI_FORMAT_UNKNOWN; + switch (idesc.Format) { + case DXGI_FORMAT_R8G8B8A8_TYPELESS: + case DXGI_FORMAT_R8G8B8A8_UNORM: + case DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: + case DXGI_FORMAT_R8G8B8A8_UINT: family = DXGI_FORMAT_R8G8B8A8_TYPELESS; break; + case DXGI_FORMAT_B8G8R8A8_TYPELESS: + case DXGI_FORMAT_B8G8R8A8_UNORM: + case DXGI_FORMAT_B8G8R8A8_UNORM_SRGB: family = DXGI_FORMAT_B8G8R8A8_TYPELESS; break; + default: break; + } + if (family == DXGI_FORMAT_UNKNOWN || idesc.SampleDesc.Count != 1 || w > idesc.Width || h > idesc.Height) { + in_tex->Release(); + LIFT_WARN_ONCE("Leia lift: unsupported input (DXGI fmt %u, %ux%u, samples %u; asked %ux%u) — need " + "single-sampled RGBA8/BGRA8", + (unsigned)idesc.Format, idesc.Width, idesc.Height, idesc.SampleDesc.Count, w, h); + return false; + } + + // ---- Bind the stream to the caller's device (first use / device change). + ID3D11Device *dev = nullptr; + in_tex->GetDevice(&dev); + if (s->dev != dev) { + release_in_bridge(s); + release_out_bridge(s); + safe_release(s->our_done); + safe_release(s->dev); + s->dev = dev; // keeps GetDevice's reference + D3D11_QUERY_DESC qd = {D3D11_QUERY_EVENT, 0}; + if (FAILED(dev->CreateQuery(&qd, &s->our_done))) { + in_tex->Release(); + LIFT_WARN_ONCE("Leia lift: CreateQuery on the caller's device failed"); + return false; + } + enable_mt_protection(ctx, "caller"); + LUID ours = {}; + if (device_luid(dev, &ours)) { + bool same = ours.HighPart == g.nd_luid.HighPart && ours.LowPart == g.nd_luid.LowPart; + U_LOG_W("Leia lift: stream %llu — caller adapter LUID %08lx:%08lx, NeurD %s (%s); bridging " + "through shared textures either way", + (unsigned long long)s->id, (unsigned long)ours.HighPart, (unsigned long)ours.LowPart, + same ? "on the SAME adapter" : "on a DIFFERENT adapter", + same ? "same-GPU copy" : "cross-adapter copy"); + } + } else { + dev->Release(); + } + + // ---- Lazily create the NeurD stream. + if (s->ns == nullptr) { + enum NeurD_status cs = NeurD_create_stream(nd, &s->ns); + if (cs != NEURD_SUCCESS || s->ns == nullptr) { + in_tex->Release(); + s->ns = nullptr; + LIFT_WARN_ONCE("Leia lift: NeurD_create_stream -> %s", status_str(cs)); + return false; + } + } + + if (!ensure_in_bridge(s, w, h, family)) { + in_tex->Release(); + return false; + } + + // ---- 1. Caller's texture -> input bridge (our device), CPU-drained. + D3D11_BOX box = {0, 0, 0, w, h, 1}; + ctx->CopySubresourceRegion(s->in_ours, 0, 0, 0, 0, in_tex, 0, &box); + in_tex->Release(); + if (!drain(ctx, s->our_done, "caller")) { + return false; + } + + // ---- 2. Pack bridge texture -> raw RGBA8 buffer (NeurD device), drained. + if (!set_cb(w, h, w * 4, 0)) { + return false; + } + g.nd_ctx->CSSetShader(g.cs_pack, nullptr, 0); + g.nd_ctx->CSSetConstantBuffers(0, 1, &g.cb); + g.nd_ctx->CSSetShaderResources(0, 1, &s->in_srv); + g.nd_ctx->CSSetUnorderedAccessViews(0, 1, &s->in_uav, nullptr); + g.nd_ctx->Dispatch(groups16(w), groups16(h), 1); + unbind_cs(); + if (!drain(g.nd_ctx, g.nd_done, "NeurD (pack)")) { + return false; + } + + // ---- 3. Global NeurD properties for this stream's mode. + struct leia_lift_neurd_params dp = {-1.0f, 1.0f, 0, 2}; + if (p != nullptr) { + dp = *p; + } + uint32_t views = 2; + uint32_t cols = 2, rows = 1; + int32_t out_type = NEURD_OUTPUT_IMAGE_TYPE_SBS; + if (s->mode == LEIA_LIFT_MODE_DEPTH) { + out_type = NEURD_OUTPUT_IMAGE_TYPE_DEPTH; + views = 1; + cols = rows = 1; + } else if (s->mode == LEIA_LIFT_MODE_NVIEW) { + // Runtime contract: N views side by side in ONE row, view 0 leftmost. + views = std::min(std::max(dp.view_count, 2u), kMaxViews); + cols = views; + rows = 1; + } + const bool auto_conv = !(dp.convergence >= 0.0f); + // strength: 1 = NeurD's calibrated budget, 0 = flat; negative/NaN = default. + // The element's strength (1 = nominal) times the depth-gain calibration: + // NeurD's own gain 1 read as too weak on the panel (David, 2026-09-26: + // strength 2 "stronger, I like this as default"). + const float strength = (dp.strength >= 0.0f) ? dp.strength : 1.0f; + const float gain = clampf(strength * l->k.depth_gain, 0.0f, 10.0f); + // NeurD always fills disocclusions; non-zero picks the blur fill, 0 the + // cheaper edge stretch. + const int32_t inpaint = (dp.inpaint != 0) ? NEURD_INPAINT_TYPE_V1_BLUR : NEURD_INPAINT_TYPE_V1_STRETCH; + + bool props_ok = prop_i(NEURD_PROP_OUTPUT_TYPE, out_type, g.p_out_type) && + prop_i(NEURD_PROP_OUTPUT_TILES_W, (int32_t)cols, g.p_tiles_w) && + prop_i(NEURD_PROP_OUTPUT_TILES_H, (int32_t)rows, g.p_tiles_h) && + prop_i(NEURD_PROP_INPAINT_TYPE, inpaint, g.p_inpaint) && + prop_i(NEURD_PROP_INPUT_AUTOSCALING, autoscale_for(s, l->k, h), + g.p_autoscale) && + prop_i(NEURD_PROP_AUTO_CONVERGENCE, auto_conv ? 1 : 0, g.p_autoconv) && + prop_f(NEURD_PROP_GAIN_MULTIPLIER, gain, g.p_gain) && + prop_i(NEURD_PROP_DILATE_RADIO, l->k.dilate, g.p_dilate); + if (props_ok && !auto_conv) { + // Runtime: convergence = RELATIVE depth placed at the display plane, + // [0,1] over the frame's depth range (0 = nearest on the glass, 1 = + // farthest). NeurD: a disparity offset in [-0.2, 0.2]. Linear map + // about the mid-range, K = DXR_LEIA_LIFT_CONV_GAIN (default 0.4 spans + // NeurD's full range; negate K if the sign proves reversed on a + // panel). UNCALIBRATED — see docs/lift-neurd.md. + const float c = clampf(dp.convergence, 0.0f, 1.0f); + const float nd_conv = clampf(l->k.conv_gain * (c - 0.5f), -0.2f, 0.2f); + props_ok = prop_f(NEURD_PROP_CONVERGENCE, nd_conv, g.p_conv); + } + g.props_valid = props_ok; + if (!props_ok) { + return false; + } + + // ---- 4. Viewpoints (SBS / NVIEW): explicit, else tracked eyes, else default. + std::vector vp; + if (s->mode != LEIA_LIFT_MODE_DEPTH) { + const uint32_t slots = cols * rows; // NeurD wants one triplet per grid tile + std::vector src_m; // metres, (x,y,z) per view + if (viewpoints_m != nullptr && viewpoint_floats >= 3 && viewpoint_floats % 3 == 0) { + const uint32_t k = viewpoint_floats / 3; + if (k == views) { + src_m.assign(viewpoints_m, viewpoints_m + viewpoint_floats); + } else if (k == 2) { + eye_left = viewpoints_m; + eye_right = viewpoints_m + 3; + eyes_valid = true; + } else { + LIFT_WARN_ONCE("Leia lift: %u explicit viewpoints for %u views — using tracked eyes", k, + views); + } + } + if (src_m.empty() && eyes_valid && eye_left != nullptr && eye_right != nullptr) { + // N views one eye-baseline apart, centred on the eye midpoint. + float c[3], d[3]; + for (int a = 0; a < 3; a++) { + c[a] = 0.5f * (eye_left[a] + eye_right[a]); + d[a] = eye_right[a] - eye_left[a]; + } + for (uint32_t i = 0; i < views; i++) { + float t = (float)i - 0.5f * (float)(views - 1); + for (int a = 0; a < 3; a++) { + src_m.push_back(c[a] + t * d[a]); + } + } + } + if (!src_m.empty()) { + vp.reserve(slots * 3); + for (uint32_t i = 0; i < slots; i++) { + uint32_t v = std::min(i, views - 1); // extra grid slots repeat the last view + float n[3]; + leia_lift_neurd_map_viewpoint(&src_m[v * 3], l->k.view_gain, n); + vp.insert(vp.end(), n, n + 3); + } + } else if (s->mode == LEIA_LIFT_MODE_NVIEW) { + // Untracked N-view: evenly spaced, one unit apart (the ±0.5 stereo + // spacing extended), centred on 0. + for (uint32_t i = 0; i < slots; i++) { + uint32_t v = std::min(i, views - 1); + vp.push_back((float)v - 0.5f * (float)(views - 1)); + vp.push_back(0.0f); + vp.push_back(0.0f); + } + } + // SBS untracked: vp stays empty -> NeurD's own default ±0.5 pattern. + + // Which viewpoints this convert used, logged once per change: an A/B of + // DXR_LEIA_LIFT_VIEW_GAIN with nobody in front of the tracker lands on the + // default pattern and reads as "the gain does nothing". + const int vp_src = !vp.empty() ? (eyes_valid ? 1 : 2) : 0; + static std::atomic s_last_vp_src{-1}; + if (s_last_vp_src.exchange(vp_src) != vp_src) { + U_LOG_W("Leia lift: viewpoints = %s (mode %d, %u view(s)%s)", + vp_src == 1 ? "TRACKED eyes" : (vp_src == 2 ? "explicit/untracked N-view" : "DEFAULT pattern (no tracked eyes)"), + (int)s->mode, views, vp_src != 0 ? ", view_gain applied" : ", view_gain NOT applied"); + } + } + + // ---- 5. Convert (blocking). + struct NeurD_image nin = {}; + nin.data = s->in_buf; + nin.width = (int32_t)w; + nin.height = (int32_t)h; + nin.stride = (int32_t)(w * 4); + nin.pix_fmt = NEURD_PIXEL_FORMAT_RGBA8; + struct NeurD_image nout = {}; + enum NeurD_status st; + if (!vp.empty() && !g.interactive_unavailable) { + st = nd_convert_stream_dx_interactive(nd, s->ns, &nin, vp.data(), (int)vp.size(), + NEURD_PIXEL_FORMAT_RGBA8, &nout); + if (st == NEURD_UNAVAILABLE_OUTDATED_RUNTIME) { + g.interactive_unavailable = true; + U_LOG_W("Leia lift: NeurD interactive convert unavailable — tracked viewpoints ignored"); + } + if (st != NEURD_SUCCESS && s->mode == LEIA_LIFT_MODE_SBS) { + LIFT_WARN_ONCE("Leia lift: interactive convert -> %s; retrying with the default pattern", + status_str(st)); + st = NeurD_convert_stream_dx(nd, s->ns, &nin, NEURD_PIXEL_FORMAT_RGBA8, &nout); + } + } else if (s->mode == LEIA_LIFT_MODE_NVIEW && g.interactive_unavailable) { + LIFT_WARN_ONCE("Leia lift: N-view needs NeurD >= 0.4.5 (interactive convert)"); + return false; + } else { + st = NeurD_convert_stream_dx(nd, s->ns, &nin, NEURD_PIXEL_FORMAT_RGBA8, &nout); + } + if (st != NEURD_SUCCESS || nout.data == nullptr) { + LIFT_WARN_ONCE("Leia lift: NeurD convert failed: %s", status_str(st)); + return false; + } + + // ---- 5b. Trust NeurD's actual layout, not the requested one: tiles in + // the one-row output = output aspect / input aspect (robust to NeurD's + // inference autoscaling). A mismatch would be mislabelled downstream. + if (nout.width <= 0 || nout.height <= 0) { + LIFT_WARN_ONCE("Leia lift: NeurD output has no size (%dx%d)", nout.width, nout.height); + return false; + } + const double tiles_f = + ((double)nout.width * (double)h) / ((double)nout.height * (double)w); + const uint32_t tiles = (uint32_t)(tiles_f + 0.5); + if (tiles != cols * rows) { + LIFT_WARN_ONCE("Leia lift: NeurD returned %dx%d (%u tile(s)) for a %ux%u input, expected %u — " + "frame dropped", + nout.width, nout.height, tiles, w, h, cols * rows); + return false; + } + + // ---- 6. Output -> bridge (NeurD device), drained; hand ours back. + if (!stage_output(s, nout) || !drain(g.nd_ctx, g.nd_done, "NeurD (unpack)")) { + return false; + } + + *out_resource = static_cast(s->out_ours); + *out_w = s->out_w; + *out_h = s->out_h; + *out_format = (uint32_t)s->out_fmt; + } catch (...) { + // std::vector / std::mutex can throw; nothing may escape into the C runtime. + LIFT_WARN_ONCE("Leia lift: exception in convert — frame dropped"); + return false; + } + + QueryPerformanceCounter(&q1); + const uint64_t ns = (uint64_t)((double)(q1.QuadPart - q0.QuadPart) * 1e9 / (double)qf.QuadPart); + const uint64_t prev = g.latency_ns.load(); + g.latency_ns.store(prev == 0 ? ns : (prev * 7 + ns) / 8); + return true; +} diff --git a/src/drv_leia/leia_lift_neurd.h b/src/drv_leia/leia_lift_neurd.h new file mode 100644 index 0000000..e5956d5 --- /dev/null +++ b/src/drv_leia/leia_lift_neurd.h @@ -0,0 +1,154 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: Apache-2.0 +/*! + * @file + * @brief NeurD-backed 2D->3D conversion ("lift") for the Leia D3D11 display + * processor. See docs/lift-neurd.md. + * + * Plug-in-owned types only — the D3D11 DP translates the runtime's + * xrt_dp_lift_* contract (behind XRT_DP_D3D11_HAS_LIFT) onto this, so the + * module builds against any runtime pin. + * + * Process model: ONE NeurD instance per process, loaded + initialised lazily on + * a background thread the first time a DP asks for caps or a stream (never at + * DP create, never on the caller's thread — first activation runs a licence + * check over the network). Each DP owns a leia_lift_neurd handle holding its + * streams; handles ref-count the instance. + * + * Absent NeurD.dll (or DXR_LEIA_LIFT=0): caps report modes=0 / state=0 and + * every other entry point returns false. Nothing is loaded. + * + * Threading: every entry point is callable from any thread. convert() is + * synchronous and blocking (tens of ms) — call it only from a worker thread, + * never a render/IPC thread. Calls are serialised process-wide (NeurD's + * properties are global). + * + * @ingroup drv_leia + */ + +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//! Mode bits (identical values to the runtime's lift contract). +#define LEIA_LIFT_MODE_DEPTH 1u +#define LEIA_LIFT_MODE_SBS 2u +#define LEIA_LIFT_MODE_NVIEW 4u + +//! Caps state (identical values to the runtime's lift contract). +#define LEIA_LIFT_STATE_UNAVAILABLE 0u +#define LEIA_LIFT_STATE_ACTIVATING 1u +#define LEIA_LIFT_STATE_READY 2u + +struct leia_lift_neurd_caps +{ + uint32_t modes; //!< LEIA_LIFT_MODE_* bits; 0 when unavailable. + uint32_t max_streams; //!< Concurrent streams (NeurD limit, process-wide). + uint32_t max_views; //!< Max view_count for NVIEW. + uint32_t depth_semantics; //!< 0 = relative (NeurD: min-max normalised disparity). + uint32_t state; //!< LEIA_LIFT_STATE_*. + uint64_t typical_latency_ns; + char backend[32]; //!< e.g. "neurd-directml"; "" when unavailable. +}; + +struct leia_lift_neurd_stream_desc +{ + uint32_t mode; //!< One LEIA_LIFT_MODE_* value. + uint32_t content_hint; //!< 0 video, 1 photo (advisory; the DX path is video-model only). + float input_scale; //!< (0,1] fraction of input height to infer at (1 = native); + //!< outside that, or DXR_LEIA_LIFT_SCALE set = the knob. +}; + +struct leia_lift_neurd_params +{ + float convergence; //!< <0 = NeurD auto-convergence; else relative depth at the display plane + //!< in [0,1], mapped to NeurD units by DXR_LEIA_LIFT_CONV_GAIN. + float strength; //!< NeurD gain multiplier: 1 = calibrated budget, 0 = flat; <0 = 1.0; max 10. + uint32_t inpaint; //!< 0 stretch fill, non-zero blur fill. + uint32_t view_count; //!< NVIEW only (2..max_views). +}; + +struct leia_lift_neurd; + +/*! + * Create a per-DP handle. Reads the env knobs (DXR_LEIA_LIFT, + * DXR_LEIA_LIFT_BACKEND, DXR_LEIA_LIFT_SCALE, DXR_LEIA_LIFT_VIEW_GAIN) once. + * Cheap: no DLL load, no thread. Never fails (a disabled handle reports + * unavailable). + */ +struct leia_lift_neurd * +leia_lift_neurd_create(void); + +void +leia_lift_neurd_destroy(struct leia_lift_neurd **lift); + +//! Non-blocking. May kick the background load/activation (or a licence retry). +bool +leia_lift_neurd_get_caps(struct leia_lift_neurd *lift, struct leia_lift_neurd_caps *out); + +//! Non-blocking. Succeeds while NeurD is still activating (the NeurD stream is +//! created lazily on the first convert); fails when unavailable / out of slots. +bool +leia_lift_neurd_stream_create(struct leia_lift_neurd *lift, + const struct leia_lift_neurd_stream_desc *desc, + uint64_t *out_id); + +void +leia_lift_neurd_stream_destroy(struct leia_lift_neurd *lift, uint64_t id); + +/*! + * Convert one frame. SYNCHRONOUS and blocking. + * + * @param d3d11_context Caller's immediate context (the device that owns + * @p input). Must be ID3D11Multithread-protected — it is + * enabled here if it is not. + * @param input ID3D11Texture2D* (as ID3D11Resource*), RGBA8 or BGRA8 + * family, single-sampled; the top-left w x h is used. + * @param viewpoints_m Optional explicit viewpoints: (x,y,z) triplets in + * METRES, display space (same space as tracked eyes). + * NULL/0 = use @p eye_left / @p eye_right. + * @param eyes_valid False (or NULL eyes) = no tracked viewer: NeurD's + * default pattern is used. + * @param out_resource ID3D11Texture2D* on the caller's device, owned by the + * stream, valid until the next convert on this stream. + * @param out_format DXGI_FORMAT: R8G8B8A8_UNORM (SBS / NVIEW, N views in one + * row, view 0 leftmost) or R8_UNORM (DEPTH). + */ +bool +leia_lift_neurd_convert(struct leia_lift_neurd *lift, + uint64_t id, + void *d3d11_context, + void *input, + uint32_t w, + uint32_t h, + const struct leia_lift_neurd_params *params, + const float *viewpoints_m, + uint32_t viewpoint_floats, + const float *eye_left, + const float *eye_right, + bool eyes_valid, + void **out_resource, + uint32_t *out_w, + uint32_t *out_h, + uint32_t *out_format); + +/*! + * Pure helper, exposed for tests + docs: map display-space positions (metres) + * to NeurD's dimensionless viewpoint units. See docs/lift-neurd.md. + * + * x_n = clamp(gain * x_m / 0.063), y_n = clamp(gain * y_m / 0.063), z_n = 0. + * A centred viewer at 63 mm IPD maps to x = -0.5 / +0.5 — exactly NeurD's + * default stereo pattern — so gain 1.0 reproduces the untracked output and + * head motion becomes look-around. + */ +void +leia_lift_neurd_map_viewpoint(const float in_m[3], float gain, float out_n[3]); + +#ifdef __cplusplus +} +#endif diff --git a/src/drv_leia/leia_plugin.c b/src/drv_leia/leia_plugin.c index bde7935..1ad54ca 100644 --- a/src/drv_leia/leia_plugin.c +++ b/src/drv_leia/leia_plugin.c @@ -437,6 +437,21 @@ static struct xrt_plugin_iface g_leia_iface = { .set_pose_source = leia_plugin_set_pose_source, .probe_displays = leia_plugin_probe_displays, + + /* + * ADR-042 lift-only D3D11 DP: no weaver / window / tracker, only the NeurD + * lift slots (docs/lift-neurd.md). Needs the runtime header's field AND the + * NeurD headers (DXR_LEIA_HAS_NEURD); otherwise the field is absent or NULL + * and the runtime has no lift from this plug-in. The iface header carrying + * this field implies its DP header carries XRT_DP_D3D11_HAS_LIFT. + */ +#if defined(XRT_PLUGIN_IFACE_HAS_D3D11_LIFT_FACTORY) +#if defined(XRT_HAVE_LEIA_SR_D3D11) && defined(DXR_LEIA_HAS_NEURD) + .create_dp_d3d11_lift = leia_dp_factory_d3d11_lift, +#else + .create_dp_d3d11_lift = NULL, +#endif +#endif };