From 9689da8cc4311c211ca2554c4101e21991064e5b Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 13:16:25 -0700 Subject: [PATCH 01/21] feat(lift): NeurD-backed 2D->3D lift module for the D3D11 DP Dynamic NeurD.dll loader (PATH -> NEURD_PATH -> HKLM\SOFTWARE\LeiaInc\NeurD), one process-wide NeurD instance activated on a background thread (licence network failure reported as ACTIVATING and retried), per-DP handle with ref-count + stream table, LeiaMeet-style device bridge (shared textures on NeurD's device, pack/unpack compute shaders, event-query drains), mode -> NeurD property mapping, tracked-eye -> NeurD viewpoint mapping. Plug-in-owned types only; the DP maps the runtime's lift contract onto it. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/drv_leia/leia_lift_neurd.cpp | 1510 ++++++++++++++++++++++++++++++ src/drv_leia/leia_lift_neurd.h | 151 +++ src/drv_leia/leia_neurd_abi.h | 223 +++++ 3 files changed, 1884 insertions(+) create mode 100644 src/drv_leia/leia_lift_neurd.cpp create mode 100644 src/drv_leia/leia_lift_neurd.h create mode 100644 src/drv_leia/leia_neurd_abi.h diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp new file mode 100644 index 0000000..cae58e0 --- /dev/null +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -0,0 +1,1510 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: Apache-2.0 +/*! + * @file + * @brief NeurD-backed 2D->3D conversion ("lift") for the Leia D3D11 DP. + * + * Design notes (full write-up: docs/lift-neurd.md): + * + * - Dynamic load only. NeurD.dll is located the way NeurD's own loader does it + * (PATH -> NEURD_PATH -> HKLM\SOFTWARE\LeiaInc\NeurD default value) but it + * is not a link dependency: a machine without NeurD loads this plug-in + * exactly as before and lift reports unavailable. + * + * - One NeurD instance per process (NeurD's rule). Loaded + initialised on a + * detached background thread, never on a caller's thread: NeurD_init runs a + * licence activation that needs the network on first use. A network failure + * is reported as ACTIVATING and retried on later calls, rate-limited. + * NeurD is never de-initialised once up: DP instances come and go on focus + * changes, and re-init re-runs licensing + model load. The ref-count only + * decides when to shrink NeurD's memory pool. + * + * - Device bridge (the LeiaMeet DxStereoConverter pattern). NeurD runs on ITS + * OWN D3D11 device (NeurD_get_dx_device), possibly on a different adapter + * than the runtime's service device on hybrid boxes. NeurD's DX input must + * be a raw RGBA8 ID3D11Buffer with D3D11_RESOURCE_MISC_SHARED, and NeurD + * hands its output back on the input buffer's device. Buffers do not share + * reliably across devices, textures do — so all buffers live on NeurD's + * device, and the crossing is done with legacy-shared TEXTURES created on + * NeurD's device and opened on ours: + * + * ours: input tex --CopySubresourceRegion--> in_bridge (opened) + * CPU: event-query drain on our context + * NeurD: in_bridge SRV --pack CS--> raw input buffer ; drain + * NeurD: convert_stream_dx[_interactive] (blocking) + * NeurD: output buffer --unpack CS--> out_bridge UAV ; drain + * ours: out_bridge (opened) is returned to the runtime + * + * Every hop is CPU-drained with an event query: NeurD's DirectML work runs + * on its own queue, unordered against any D3D11 context. + * + * - NeurD properties are process-global, so the (set props -> convert) pair + * must be atomic across streams: one process-wide mutex serialises every + * NeurD call and every use of NeurD's immediate context. + * + * - Never throws, never blocks the caller on activation; errors return false + * with a WARN once per site. + */ + +#include "leia_lift_neurd.h" +#include "leia_neurd_abi.h" + +#include "util/u_logging.h" + +#include +#include +#include // ID3D11Multithread +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +/* + * + * Constants + small helpers. + * + */ + +//! Reference inter-pupillary distance: NeurD's default stereo pattern places +//! the two views at x = -0.5 / +0.5, i.e. ONE unit of x == one "nominal eye +//! baseline". Mapping metres through this IPD makes a centred, tracked viewer +//! reproduce the default pattern exactly. +constexpr float kIpdRefM = 0.063f; +//! Clamp for mapped viewpoints — well past any comfortable look-around; stops a +//! tracker glitch from asking NeurD for a wildly extrapolated view. +constexpr float kViewpointClamp = 3.0f; +//! NeurD's MAX_STREAMS. +constexpr uint32_t kMaxStreams = 32; +constexpr uint32_t kMaxViews = 16; +//! Retry period for a licence activation that failed on the network. +constexpr uint64_t kActivationRetryMs = 15000; +//! Upper bound for any CPU drain of a GPU queue (device-lost guard). +constexpr uint64_t kDrainTimeoutMs = 2000; +//! Latency priors (reported until a measurement exists). +constexpr uint64_t kPriorLatencyDirectMlNs = 22000000ull; +constexpr uint64_t kPriorLatencyCudaNs = 14000000ull; +constexpr uint64_t kPriorLatencyOtherNs = 40000000ull; + +#define LIFT_WARN_ONCE(...) \ + do { \ + static std::atomic warned_{false}; \ + if (!warned_.exchange(true)) { \ + U_LOG_W(__VA_ARGS__); \ + } \ + } while (0) + +template +void +safe_release(T *&p) +{ + if (p != nullptr) { + p->Release(); + p = nullptr; + } +} + +inline float +clampf(float v, float lo, float hi) +{ + return v < lo ? lo : (v > hi ? hi : v); +} + +const char * +status_str(enum leia_neurd_status s) +{ + switch (s) { + case LEIA_NEURD_SUCCESS: return "SUCCESS"; + case LEIA_NEURD_IN_PROGRESS: return "IN_PROGRESS"; + case LEIA_NEURD_GENERIC_ERROR: return "GENERIC_ERROR"; + case LEIA_NEURD_INVALID_ARG: return "INVALID_ARG"; + case LEIA_NEURD_INVALID_LICENSE: return "INVALID_LICENSE"; + case LEIA_NEURD_UNAVAILABLE_OUTDATED_RUNTIME: return "UNAVAILABLE_OUTDATED_RUNTIME"; + case LEIA_NEURD_NOT_INITIALIZED: return "NOT_INITIALIZED"; + case LEIA_NEURD_LICENSE_NETWORK_ERROR: return "LICENSE_NETWORK_ERROR"; + case LEIA_NEURD_INVALID_MODEL: return "INVALID_MODEL"; + case LEIA_NEURD_FILESYSTEM_ERROR: return "FILESYSTEM_ERROR"; + case LEIA_NEURD_NETWORK_ERROR: return "NETWORK_ERROR"; + case LEIA_NEURD_ABORTED: return "ABORTED"; + default: return "UNKNOWN"; + } +} + +const char * +backend_str(enum leia_neurd_backend b) +{ + switch (b) { + case LEIA_NEURD_BACKEND_CUDA: return "cuda"; + case LEIA_NEURD_BACKEND_DIRECTML: return "directml"; + case LEIA_NEURD_BACKEND_OPENVINO: return "openvino"; + default: return "unknown"; + } +} + +/* + * + * Env knobs (read once per DP handle, at leia_lift_neurd_create). + * + */ + +enum backend_choice +{ + BACKEND_AUTO = -1, + BACKEND_CUDA = LEIA_NEURD_BACKEND_CUDA, + BACKEND_DIRECTML = LEIA_NEURD_BACKEND_DIRECTML, + BACKEND_OPENVINO = LEIA_NEURD_BACKEND_OPENVINO, +}; + +struct knobs +{ + bool enabled; //!< DXR_LEIA_LIFT (default on) + int backend; //!< DXR_LEIA_LIFT_BACKEND (default directml) + int32_t autoscaling; //!< DXR_LEIA_LIFT_SCALE (default 720p) + float view_gain; //!< DXR_LEIA_LIFT_VIEW_GAIN (default 1.0) +}; + +bool +env_ieq(const char *a, const char *b) +{ + return _stricmp(a, b) == 0; +} + +struct knobs +read_knobs() +{ + struct knobs k = {}; + k.enabled = true; + k.backend = BACKEND_DIRECTML; + k.autoscaling = LEIA_NEURD_AUTOSCALING_720P; + k.view_gain = 1.0f; + + const char *e = std::getenv("DXR_LEIA_LIFT"); + if (e != nullptr && (e[0] == '0' || env_ieq(e, "off") || env_ieq(e, "false"))) { + k.enabled = false; + } + + e = std::getenv("DXR_LEIA_LIFT_BACKEND"); + if (e != nullptr && e[0] != '\0') { + if (env_ieq(e, "auto")) { + k.backend = BACKEND_AUTO; + } else if (env_ieq(e, "cuda")) { + k.backend = BACKEND_CUDA; + } else if (env_ieq(e, "directml") || env_ieq(e, "dml")) { + k.backend = BACKEND_DIRECTML; + } else if (env_ieq(e, "openvino")) { + k.backend = BACKEND_OPENVINO; + } else { + U_LOG_W("Leia lift: DXR_LEIA_LIFT_BACKEND='%s' not recognised (auto|directml|cuda|openvino) — " + "using directml", + e); + } + } + + e = std::getenv("DXR_LEIA_LIFT_SCALE"); + if (e != nullptr && e[0] != '\0') { + if (env_ieq(e, "none") || env_ieq(e, "native") || env_ieq(e, "0")) { + k.autoscaling = LEIA_NEURD_AUTOSCALING_NONE; + } else if (env_ieq(e, "720") || env_ieq(e, "720p")) { + k.autoscaling = LEIA_NEURD_AUTOSCALING_720P; + } else if (env_ieq(e, "1080") || env_ieq(e, "1080p")) { + k.autoscaling = LEIA_NEURD_AUTOSCALING_1080P; + } else if (env_ieq(e, "1440") || env_ieq(e, "1440p")) { + k.autoscaling = LEIA_NEURD_AUTOSCALING_1440P; + } else { + U_LOG_W("Leia lift: DXR_LEIA_LIFT_SCALE='%s' not recognised (720|1080|1440|none) — using 720", e); + } + } + + e = std::getenv("DXR_LEIA_LIFT_VIEW_GAIN"); + if (e != nullptr && e[0] != '\0') { + float g = (float)std::atof(e); + if (std::isfinite(g) && g >= 0.0f && g <= 10.0f) { + k.view_gain = g; + } else { + U_LOG_W("Leia lift: DXR_LEIA_LIFT_VIEW_GAIN='%s' out of range [0,10] — using 1.0", e); + } + } + return k; +} + +/* + * + * Compute shaders for the bridge (compiled at runtime on NeurD's device). + * + */ + +// Bridge texture (typed RGBA8/BGRA8 UNORM view — the view's format swizzles, so +// .rgba is always R,G,B,A) -> tightly packed RGBA8 raw buffer. +const char *kPackCs = R"( +cbuffer P : register(b0) { uint W; uint H; uint StrideBytes; uint Pad; }; +Texture2D Src : register(t0); +RWByteAddressBuffer Dst : register(u0); +[numthreads(16, 16, 1)] +void main(uint3 id : SV_DispatchThreadID) +{ + if (id.x >= W || id.y >= H) return; + uint4 u = (uint4)round(saturate(Src.Load(int3(id.xy, 0))) * 255.0); + Dst.Store(id.y * StrideBytes + id.x * 4, u.r | (u.g << 8) | (u.b << 16) | (u.a << 24)); +} +)"; + +// NeurD's RGBA8 raw output buffer -> bridge output texture. +const char *kUnpackCs = R"( +cbuffer P : register(b0) { uint W; uint H; uint StrideBytes; uint Pad; }; +RWByteAddressBuffer Src : register(u0); +RWTexture2D Dst : register(u1); +[numthreads(16, 16, 1)] +void main(uint3 id : SV_DispatchThreadID) +{ + if (id.x >= W || id.y >= H) return; + uint v = Src.Load(id.y * StrideBytes + id.x * 4); + Dst[id.xy] = float4(v & 0xFF, (v >> 8) & 0xFF, (v >> 16) & 0xFF, (v >> 24) & 0xFF) / 255.0; +} +)"; + +/* + * + * Process-wide NeurD instance. + * + */ + +enum gstate : uint32_t +{ + G_UNPROBED = 0, //!< Nothing looked at yet. + G_ABSENT, //!< No NeurD.dll found — permanent for the process. + G_LOADING, //!< Background worker is loading / initialising. + G_RETRY_WAIT, //!< Licence activation hit the network; retry after next_retry_ms. + G_READY, //!< Up; converts allowed. + G_FAILED, //!< Permanent failure (bad licence, no DX device, ABI...). +}; + +struct global +{ + std::mutex mtx; //!< Serialises every NeurD call + NeurD-context use + the fields below. + + std::atomic state{G_UNPROBED}; + std::atomic worker_running{false}; + std::atomic next_retry_ms{0}; + std::atomic latency_ns{0}; //!< EMA of full convert wall time; 0 = none yet. + std::atomic streams_live{0}; + + int requested_backend = BACKEND_DIRECTML; //!< First acquirer's choice (NeurD's forced backend is sticky). + int32_t default_autoscaling = LEIA_NEURD_AUTOSCALING_720P; + + HMODULE lib = nullptr; + const struct leia_neurd_table *nd = nullptr; + enum leia_neurd_backend backend = LEIA_NEURD_BACKEND_DIRECTML; + char backend_name[32] = {}; + + ID3D11Device *nd_dev = nullptr; //!< NeurD-owned: never Released. + ID3D11DeviceContext *nd_ctx = nullptr; + ID3D11Query *nd_done = nullptr; + ID3D11ComputeShader *cs_pack = nullptr; + ID3D11ComputeShader *cs_unpack = nullptr; + ID3D11Buffer *cb = nullptr; + LUID nd_luid = {}; + + int refcount = 0; + uint64_t next_stream_id = 1; + bool interactive_unavailable = false; + + //! Last-applied global NeurD properties (avoid a worker round-trip per prop per frame). + bool props_valid = false; + int32_t p_out_type = -1, p_tiles_w = -1, p_tiles_h = -1, p_inpaint = -1, p_autoscale = -1, p_autoconv = -1; + float p_conv = -1.0f, p_gain = -1.0f; +}; + +global g; + +uint64_t +now_ms() +{ + return GetTickCount64(); +} + +/* + * DLL location — mirrors NeurD's own loader order. Presence probe is separate + * from the load so caps can say "activating" without pulling NeurD (and its + * CUDA/DirectML/OpenVINO dependency tree) into the process on the caller's + * thread. + */ +bool +registry_neurd_path(char *out, size_t cap) +{ + HKEY key = nullptr; + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "SOFTWARE\\LeiaInc\\NeurD", 0, KEY_READ, &key) != ERROR_SUCCESS) { + return false; + } + char dir[MAX_PATH] = {}; + DWORD sz = sizeof(dir); + LSTATUS r = RegGetValueA(key, nullptr, nullptr, RRF_RT_REG_SZ, nullptr, dir, &sz); + RegCloseKey(key); + if (r != ERROR_SUCCESS || dir[0] == '\0') { + return false; + } + size_t n = strlen(dir); + const char *sep = (dir[n - 1] == '\\' || dir[n - 1] == '/') ? "" : "\\"; + int w = snprintf(out, cap, "%s%sNeurD.dll", dir, sep); + return w > 0 && (size_t)w < cap; +} + +bool +file_exists(const char *path) +{ + DWORD a = GetFileAttributesA(path); + return a != INVALID_FILE_ATTRIBUTES && (a & FILE_ATTRIBUTE_DIRECTORY) == 0; +} + +bool +neurd_present() +{ + char buf[MAX_PATH]; + if (SearchPathA(nullptr, "NeurD.dll", nullptr, MAX_PATH, buf, nullptr) != 0) { + return true; + } + const char *np = std::getenv("NEURD_PATH"); + if (np != nullptr && np[0] != '\0' && file_exists(np)) { + return true; + } + return registry_neurd_path(buf, sizeof(buf)) && file_exists(buf); +} + +HMODULE +neurd_load_library(const char **out_where) +{ + HMODULE h = LoadLibraryA("NeurD.dll"); + if (h != nullptr) { + *out_where = "PATH"; + return h; + } + const char *np = std::getenv("NEURD_PATH"); + if (np != nullptr && np[0] != '\0') { + h = LoadLibraryExA(np, nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); + if (h != nullptr) { + *out_where = "NEURD_PATH"; + return h; + } + } + char path[MAX_PATH]; + if (registry_neurd_path(path, sizeof(path))) { + h = LoadLibraryExA(path, nullptr, LOAD_WITH_ALTERED_SEARCH_PATH); + if (h != nullptr) { + *out_where = "HKLM\\SOFTWARE\\LeiaInc\\NeurD"; + return h; + } + } + return nullptr; +} + +void +neurd_log_cb(const char *msg, int size) +{ + if (msg == nullptr || size <= 0) { + return; + } + // Not NUL-terminated (NeurD contract). Trim a trailing newline. + while (size > 0 && (msg[size - 1] == '\n' || msg[size - 1] == '\r')) { + size--; + } + U_LOG_I("NeurD: %.*s", size, msg); +} + +bool +enable_mt_protection(ID3D11DeviceContext *ctx, const char *who) +{ + ID3D11Multithread *mt = nullptr; + if (FAILED(ctx->QueryInterface(__uuidof(ID3D11Multithread), (void **)&mt)) || mt == nullptr) { + U_LOG_W("Leia lift: %s context has no ID3D11Multithread — concurrent use is unsafe", who); + return false; + } + if (!mt->GetMultithreadProtected()) { + mt->SetMultithreadProtected(TRUE); + U_LOG_W("Leia lift: enabled ID3D11Multithread protection on the %s immediate context", who); + } + mt->Release(); + return true; +} + +bool +device_luid(ID3D11Device *dev, LUID *out) +{ + IDXGIDevice *dxgi = nullptr; + if (FAILED(dev->QueryInterface(__uuidof(IDXGIDevice), (void **)&dxgi)) || dxgi == nullptr) { + return false; + } + IDXGIAdapter *ad = nullptr; + bool ok = false; + if (SUCCEEDED(dxgi->GetAdapter(&ad)) && ad != nullptr) { + DXGI_ADAPTER_DESC d = {}; + if (SUCCEEDED(ad->GetDesc(&d))) { + *out = d.AdapterLuid; + ok = true; + } + ad->Release(); + } + dxgi->Release(); + return ok; +} + +ID3D11ComputeShader * +compile_cs(ID3D11Device *dev, const char *src, const char *name) +{ + ID3DBlob *blob = nullptr; + ID3DBlob *err = nullptr; + HRESULT hr = D3DCompile(src, strlen(src), name, nullptr, nullptr, "main", "cs_5_0", 0, 0, &blob, &err); + if (FAILED(hr)) { + U_LOG_E("Leia lift: %s compile failed: %s", name, + err != nullptr ? (const char *)err->GetBufferPointer() : "(no log)"); + safe_release(err); + return nullptr; + } + safe_release(err); + ID3D11ComputeShader *cs = nullptr; + hr = dev->CreateComputeShader(blob->GetBufferPointer(), blob->GetBufferSize(), nullptr, &cs); + blob->Release(); + if (FAILED(hr)) { + U_LOG_E("Leia lift: CreateComputeShader(%s) failed: 0x%08x", name, (unsigned)hr); + return nullptr; + } + return cs; +} + +//! Build the NeurD-device side of the bridge. Called with g.mtx held. +bool +setup_nd_device_locked() +{ + g.nd_dev = static_cast(g.nd->get_dx_device()); + if (g.nd_dev == nullptr) { + U_LOG_W("Leia lift: NeurD backend '%s' exposes no D3D11 device — the D3D11 lift path needs the " + "DirectML backend (DXR_LEIA_LIFT_BACKEND=directml). Lift unavailable.", + g.backend_name); + return false; + } + g.nd_dev->GetImmediateContext(&g.nd_ctx); + enable_mt_protection(g.nd_ctx, "NeurD"); + + D3D11_QUERY_DESC qd = {D3D11_QUERY_EVENT, 0}; + g.cs_pack = compile_cs(g.nd_dev, kPackCs, "leia_lift_pack"); + g.cs_unpack = compile_cs(g.nd_dev, kUnpackCs, "leia_lift_unpack"); + D3D11_BUFFER_DESC bd = {}; + bd.ByteWidth = 16; + bd.Usage = D3D11_USAGE_DYNAMIC; + bd.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + bd.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + if (g.cs_pack == nullptr || g.cs_unpack == nullptr || FAILED(g.nd_dev->CreateQuery(&qd, &g.nd_done)) || + FAILED(g.nd_dev->CreateBuffer(&bd, nullptr, &g.cb))) { + U_LOG_W("Leia lift: bridge setup on NeurD's device failed — lift unavailable"); + safe_release(g.cs_pack); + safe_release(g.cs_unpack); + safe_release(g.nd_done); + safe_release(g.cb); + safe_release(g.nd_ctx); + g.nd_dev = nullptr; + return false; + } + if (device_luid(g.nd_dev, &g.nd_luid)) { + U_LOG_W("Leia lift: NeurD D3D11 device on adapter LUID %08lx:%08lx (FL 0x%x)", + (unsigned long)g.nd_luid.HighPart, (unsigned long)g.nd_luid.LowPart, + (unsigned)g.nd_dev->GetFeatureLevel()); + } + return true; +} + +/*! + * Background worker: load (once) + init (retryable). Runs detached; publishes + * the outcome through g.state. + */ +void +activation_worker() +{ + uint32_t next = G_FAILED; + { + // Load happens once per process. NOT under g.mtx for the slow part — + // but nothing else touches g.lib / g.nd until state says READY. + if (g.nd == nullptr) { + const char *where = "?"; + HMODULE lib = neurd_load_library(&where); + if (lib == nullptr) { + U_LOG_W("Leia lift: NeurD.dll present but LoadLibrary failed (err %lu) — lift unavailable", + (unsigned long)GetLastError()); + g.state.store(G_FAILED); + g.worker_running.store(false); + return; + } + auto load = reinterpret_cast(reinterpret_cast(GetProcAddress(lib, "NeurD_load"))); + struct leia_neurd_load_request req = {LEIA_NEURD_BUILT_AGAINST}; + const struct leia_neurd_table *nd = load != nullptr ? load(&req) : nullptr; + if (nd == nullptr) { + U_LOG_W("Leia lift: NeurD.dll (%s) has no usable NeurD_load — lift unavailable", where); + // Deliberately no FreeLibrary: NeurD may have spun threads. + g.state.store(G_FAILED); + g.worker_running.store(false); + return; + } + U_LOG_W("Leia lift: loaded NeurD %u.%u.%u from %s (plug-in built against %u.%u.%u)", + LEIA_NEURD_VERSION_MAJOR(nd->version), LEIA_NEURD_VERSION_MINOR(nd->version), + LEIA_NEURD_VERSION_PATCH(nd->version), where, + LEIA_NEURD_VERSION_MAJOR(LEIA_NEURD_BUILT_AGAINST), + LEIA_NEURD_VERSION_MINOR(LEIA_NEURD_BUILT_AGAINST), + LEIA_NEURD_VERSION_PATCH(LEIA_NEURD_BUILT_AGAINST)); + if (LEIA_NEURD_VERSION_MAJOR(nd->version) != 0 || + !LEIA_NEURD_HAS(nd, convert_stream_dx, 0, 3, 11) || !LEIA_NEURD_HAS(nd, get_dx_device, 0, 4, 3) || + !LEIA_NEURD_HAS(nd, create_stream, 0, 3, 11) || !LEIA_NEURD_HAS(nd, set_prop_1i, 0, 2, 1) || + !LEIA_NEURD_HAS(nd, set_prop_1f, 0, 2, 1)) { + U_LOG_W("Leia lift: NeurD runtime too old/new for the D3D11 stream path (need 0.4.3+, " + "major 0) — lift unavailable"); + g.state.store(G_FAILED); + g.worker_running.store(false); + return; + } + g.lib = lib; + g.nd = nd; + if (LEIA_NEURD_HAS(nd, set_logger_callback, 0, 2, 5)) { + nd->set_logger_callback(neurd_log_cb); + } + } + + const struct leia_neurd_table *nd = g.nd; + + // Properties that must precede init (NeurD reads some at init time): + // the defaults this plug-in wants for every stream. + nd->set_prop_1i(LEIA_NEURD_PROP_OUTPUT_TYPE, LEIA_NEURD_OUTPUT_SBS); + nd->set_prop_1i(LEIA_NEURD_PROP_INPUT_AUTOSCALING, g.default_autoscaling); + nd->set_prop_1i(LEIA_NEURD_PROP_INPAINT_TYPE, LEIA_NEURD_INPAINT_V1_STRETCH); + + // Backend must be forced immediately before init. NeurD keeps a forced + // backend across deinit, so this is effectively set once per process. + if (g.requested_backend != BACKEND_AUTO && LEIA_NEURD_HAS(nd, set_backend, 0, 3, 4)) { + enum leia_neurd_status bs = nd->set_backend((enum leia_neurd_backend)g.requested_backend); + if (bs != LEIA_NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD_set_backend(%s) -> %s", backend_str((enum leia_neurd_backend)g.requested_backend), + status_str(bs)); + } + } + + enum leia_neurd_status st; + if (LEIA_NEURD_HAS(nd, init_with_options, 0, 4, 6)) { + struct leia_neurd_init_options opts = {}; + opts.struct_size = sizeof(opts); + opts.photo_model_id = LEIA_NEURD_MODEL_PHOTO_RELATIVE_QUALITY; + opts.video_model_id = LEIA_NEURD_MODEL_VIDEO_RELATIVE_FAST; + st = nd->init_with_options(1 /* multithreaded: we call from the runtime's lift thread */, &opts); + } else if (LEIA_NEURD_HAS(nd, init, 0, 2, 2)) { + st = nd->init(1); + } else { + st = LEIA_NEURD_UNAVAILABLE_OUTDATED_RUNTIME; + } + + if (st == LEIA_NEURD_LICENSE_NETWORK_ERROR || st == LEIA_NEURD_NETWORK_ERROR) { + g.next_retry_ms.store(now_ms() + kActivationRetryMs); + U_LOG_W("Leia lift: NeurD licence activation needs the network (%s) — reporting ACTIVATING, " + "retrying in %llus", + status_str(st), (unsigned long long)(kActivationRetryMs / 1000)); + g.state.store(G_RETRY_WAIT); + g.worker_running.store(false); + return; + } + if (st != LEIA_NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD init failed: %s — lift unavailable", status_str(st)); + g.state.store(G_FAILED); + g.worker_running.store(false); + return; + } + + enum leia_neurd_backend be = (g.requested_backend == BACKEND_AUTO) + ? LEIA_NEURD_BACKEND_DIRECTML + : (enum leia_neurd_backend)g.requested_backend; + if (LEIA_NEURD_HAS(nd, get_backend, 0, 3, 4)) { + nd->get_backend(&be); + } + + std::lock_guard lock(g.mtx); + g.backend = be; + snprintf(g.backend_name, sizeof(g.backend_name), "neurd-%s", backend_str(be)); + g.interactive_unavailable = !LEIA_NEURD_HAS(nd, convert_stream_dx_interactive, 0, 4, 5); + g.props_valid = false; + next = setup_nd_device_locked() ? (uint32_t)G_READY : (uint32_t)G_FAILED; + if (next == G_READY) { + U_LOG_W("Leia lift: NeurD READY — backend %s, interactive viewpoints %s", g.backend_name, + g.interactive_unavailable ? "UNAVAILABLE (NeurD < 0.4.5)" : "available"); + } + } + g.state.store(next); + g.worker_running.store(false); +} + +/*! + * Advance the process state machine without blocking: probe presence on first + * touch, and (re)start the activation worker when due. + */ +void +kick(int requested_backend, int32_t default_autoscaling) +{ + uint32_t s = g.state.load(); + if (s == G_UNPROBED) { + // Racing first callers are fine: presence is idempotent and only one + // wins the worker_running exchange below. + if (!neurd_present()) { + uint32_t expect = G_UNPROBED; + if (g.state.compare_exchange_strong(expect, G_ABSENT)) { + U_LOG_W("Leia lift: NeurD.dll not found (PATH / NEURD_PATH / HKLM\\SOFTWARE\\LeiaInc\\NeurD) — " + "2D->3D lift unavailable; plug-in otherwise unaffected"); + } + return; + } + } else if (s == G_RETRY_WAIT) { + if (now_ms() < g.next_retry_ms.load()) { + return; + } + } else { + return; // ABSENT / LOADING / READY / FAILED: nothing to do. + } + + bool expect_idle = false; + if (!g.worker_running.compare_exchange_strong(expect_idle, true)) { + return; + } + if (s == G_UNPROBED) { + g.requested_backend = requested_backend; + g.default_autoscaling = default_autoscaling; + } + g.state.store(G_LOADING); + try { + std::thread(activation_worker).detach(); + } catch (...) { + U_LOG_W("Leia lift: could not start the NeurD activation thread"); + g.state.store(s); + g.worker_running.store(false); + } +} + +uint32_t +public_state(uint32_t s) +{ + switch (s) { + case G_READY: return LEIA_LIFT_STATE_READY; + case G_UNPROBED: + case G_LOADING: + case G_RETRY_WAIT: return LEIA_LIFT_STATE_ACTIVATING; + default: return LEIA_LIFT_STATE_UNAVAILABLE; + } +} + +/* + * + * Per-stream bridge. + * + */ + +struct lift_stream +{ + uint64_t id; + uint32_t mode; + uint32_t content_hint; + float input_scale; + struct leia_neurd_stream *ns; //!< Created lazily on first convert (NeurD must be READY). + + ID3D11Device *dev; //!< Caller's device (AddRef'd) the bridges are opened on. + ID3D11Query *our_done; + + // Input bridge. + uint32_t in_w, in_h; + DXGI_FORMAT in_family; //!< R8G8B8A8_TYPELESS or B8G8R8A8_TYPELESS. + ID3D11Texture2D *in_nd; + ID3D11Texture2D *in_ours; + ID3D11ShaderResourceView *in_srv; + ID3D11Buffer *in_buf; + ID3D11UnorderedAccessView *in_uav; + + // Output bridge. + uint32_t out_w, out_h; + ID3D11Texture2D *out_nd; + ID3D11Texture2D *out_ours; + ID3D11UnorderedAccessView *out_uav; + ID3D11Resource *out_src_key; //!< NeurD output resource the raw UAV below was made for. + ID3D11UnorderedAccessView *out_src_uav; +}; + +void +release_in_bridge(lift_stream *s) +{ + safe_release(s->in_uav); + safe_release(s->in_buf); + safe_release(s->in_srv); + safe_release(s->in_ours); + safe_release(s->in_nd); + s->in_w = s->in_h = 0; +} + +void +release_out_bridge(lift_stream *s) +{ + safe_release(s->out_src_uav); + s->out_src_key = nullptr; // not owned + safe_release(s->out_uav); + safe_release(s->out_ours); + safe_release(s->out_nd); + s->out_w = s->out_h = 0; +} + +//! Called with g.mtx held. +void +release_stream_locked(lift_stream *s) +{ + release_in_bridge(s); + release_out_bridge(s); + safe_release(s->our_done); + safe_release(s->dev); + if (s->ns != nullptr && g.state.load() == G_READY && LEIA_NEURD_HAS(g.nd, destroy_stream, 0, 3, 11)) { + g.nd->destroy_stream(s->ns); + } + s->ns = nullptr; +} + +/*! + * Shared texture on NeurD's device, opened on the caller's device. Legacy + * (non-NT) handle — the one NeurD itself and LeiaMeet rely on; not owned, no + * CloseHandle. + */ +bool +make_shared_tex(ID3D11Device *ours, uint32_t w, uint32_t h, DXGI_FORMAT fmt, UINT bind, ID3D11Texture2D **out_nd, + ID3D11Texture2D **out_ours) +{ + D3D11_TEXTURE2D_DESC td = {}; + td.Width = w; + td.Height = h; + td.MipLevels = 1; + td.ArraySize = 1; + td.Format = fmt; + td.SampleDesc.Count = 1; + td.Usage = D3D11_USAGE_DEFAULT; + td.BindFlags = bind; + td.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + if (FAILED(g.nd_dev->CreateTexture2D(&td, nullptr, out_nd))) { + return false; + } + IDXGIResource *dr = nullptr; + HANDLE hshared = nullptr; + bool ok = SUCCEEDED((*out_nd)->QueryInterface(__uuidof(IDXGIResource), (void **)&dr)) && dr != nullptr && + SUCCEEDED(dr->GetSharedHandle(&hshared)) && hshared != nullptr && + SUCCEEDED(ours->OpenSharedResource(hshared, __uuidof(ID3D11Texture2D), (void **)out_ours)); + safe_release(dr); + if (!ok) { + safe_release(*out_nd); + return false; + } + return true; +} + +bool +ensure_in_bridge(lift_stream *s, uint32_t w, uint32_t h, DXGI_FORMAT family) +{ + if (s->in_nd != nullptr && s->in_w == w && s->in_h == h && s->in_family == family) { + return true; + } + release_in_bridge(s); + if (!make_shared_tex(s->dev, w, h, family, D3D11_BIND_SHADER_RESOURCE, &s->in_nd, &s->in_ours)) { + LIFT_WARN_ONCE("Leia lift: input bridge texture %ux%u create/open failed", w, h); + return false; + } + D3D11_SHADER_RESOURCE_VIEW_DESC sd = {}; + sd.Format = (family == DXGI_FORMAT_B8G8R8A8_TYPELESS) ? DXGI_FORMAT_B8G8R8A8_UNORM : DXGI_FORMAT_R8G8B8A8_UNORM; + sd.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + sd.Texture2D.MipLevels = 1; + + // Raw input buffer NeurD accepts: flat RGBA8, stride w*4, SHARED. + D3D11_BUFFER_DESC bd = {}; + bd.ByteWidth = w * h * 4; + bd.Usage = D3D11_USAGE_DEFAULT; + bd.BindFlags = D3D11_BIND_UNORDERED_ACCESS | D3D11_BIND_SHADER_RESOURCE; + bd.MiscFlags = D3D11_RESOURCE_MISC_BUFFER_ALLOW_RAW_VIEWS | D3D11_RESOURCE_MISC_SHARED; + D3D11_UNORDERED_ACCESS_VIEW_DESC ud = {}; + ud.Format = DXGI_FORMAT_R32_TYPELESS; + ud.ViewDimension = D3D11_UAV_DIMENSION_BUFFER; + ud.Buffer.NumElements = w * h; + ud.Buffer.Flags = D3D11_BUFFER_UAV_FLAG_RAW; + + if (FAILED(g.nd_dev->CreateShaderResourceView(s->in_nd, &sd, &s->in_srv)) || + FAILED(g.nd_dev->CreateBuffer(&bd, nullptr, &s->in_buf)) || + FAILED(g.nd_dev->CreateUnorderedAccessView(s->in_buf, &ud, &s->in_uav))) { + LIFT_WARN_ONCE("Leia lift: input bridge views/buffer %ux%u failed", w, h); + release_in_bridge(s); + return false; + } + s->in_w = w; + s->in_h = h; + s->in_family = family; + return true; +} + +bool +ensure_out_bridge(lift_stream *s, uint32_t w, uint32_t h) +{ + if (s->out_nd != nullptr && s->out_w == w && s->out_h == h) { + return true; + } + release_out_bridge(s); + if (!make_shared_tex(s->dev, w, h, DXGI_FORMAT_R8G8B8A8_UNORM, + D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_UNORDERED_ACCESS, &s->out_nd, &s->out_ours) || + FAILED(g.nd_dev->CreateUnorderedAccessView(s->out_nd, nullptr, &s->out_uav))) { + LIFT_WARN_ONCE("Leia lift: output bridge texture %ux%u failed", w, h); + release_out_bridge(s); + return false; + } + s->out_w = w; + s->out_h = h; + return true; +} + +//! Spin until @p q (an EVENT query already End()ed on @p ctx) signals. +bool +drain(ID3D11DeviceContext *ctx, ID3D11Query *q, const char *who) +{ + ctx->End(q); + ctx->Flush(); + const uint64_t t0 = now_ms(); + HRESULT hr; + while ((hr = ctx->GetData(q, nullptr, 0, 0)) == S_FALSE) { + if (now_ms() - t0 > kDrainTimeoutMs) { + U_LOG_W("Leia lift: %s queue drain timed out (%llu ms) — device hung/lost?", who, + (unsigned long long)kDrainTimeoutMs); + return false; + } + std::this_thread::yield(); + } + if (hr != S_OK) { + U_LOG_W("Leia lift: %s queue drain failed 0x%08x — device lost?", who, (unsigned)hr); + return false; + } + return true; +} + +bool +set_cb(uint32_t a, uint32_t b, uint32_t c, uint32_t d) +{ + D3D11_MAPPED_SUBRESOURCE m = {}; + if (FAILED(g.nd_ctx->Map(g.cb, 0, D3D11_MAP_WRITE_DISCARD, 0, &m))) { + return false; + } + uint32_t *p = static_cast(m.pData); + p[0] = a; + p[1] = b; + p[2] = c; + p[3] = d; + g.nd_ctx->Unmap(g.cb, 0); + return true; +} + +void +unbind_cs() +{ + ID3D11UnorderedAccessView *nu[2] = {nullptr, nullptr}; + ID3D11ShaderResourceView *ns = nullptr; + g.nd_ctx->CSSetUnorderedAccessViews(0, 2, nu, nullptr); + g.nd_ctx->CSSetShaderResources(0, 1, &ns); + g.nd_ctx->CSSetShader(nullptr, nullptr, 0); +} + +inline UINT +groups16(uint32_t n) +{ + return (n + 15) / 16; +} + +//! Copy/unpack NeurD's output into the stream's output bridge (NeurD device). +bool +stage_output(lift_stream *s, const struct leia_neurd_image &out) +{ + auto *res = static_cast(out.data); + if (res == nullptr || out.width <= 0 || out.height <= 0) { + return false; + } + const uint32_t w = (uint32_t)out.width; + const uint32_t h = (uint32_t)out.height; + if (!ensure_out_bridge(s, w, h)) { + return false; + } + + ID3D11Texture2D *tex = nullptr; + if (SUCCEEDED(res->QueryInterface(__uuidof(ID3D11Texture2D), (void **)&tex)) && tex != nullptr) { + // Region copy: NeurD's internal texture may exceed the reported size. + D3D11_BOX box = {0, 0, 0, w, h, 1}; + g.nd_ctx->CopySubresourceRegion(s->out_nd, 0, 0, 0, 0, tex, 0, &box); + tex->Release(); + return true; + } + + ID3D11Buffer *buf = nullptr; + if (FAILED(res->QueryInterface(__uuidof(ID3D11Buffer), (void **)&buf)) || buf == nullptr) { + LIFT_WARN_ONCE("Leia lift: NeurD output is neither a texture nor a buffer"); + return false; + } + D3D11_BUFFER_DESC bd = {}; + buf->GetDesc(&bd); + uint32_t stride = (out.stride >= (int32_t)(w * 4)) ? (uint32_t)out.stride : w * 4; + if ((uint64_t)stride * h > bd.ByteWidth) { + buf->Release(); + LIFT_WARN_ONCE("Leia lift: NeurD output buffer too small (%u < %llu bytes)", bd.ByteWidth, + (unsigned long long)stride * h); + return false; + } + if (s->out_src_key != res || s->out_src_uav == nullptr) { + safe_release(s->out_src_uav); + D3D11_UNORDERED_ACCESS_VIEW_DESC ud = {}; + ud.Format = DXGI_FORMAT_R32_TYPELESS; + ud.ViewDimension = D3D11_UAV_DIMENSION_BUFFER; + ud.Buffer.NumElements = bd.ByteWidth / 4; + ud.Buffer.Flags = D3D11_BUFFER_UAV_FLAG_RAW; + if (FAILED(g.nd_dev->CreateUnorderedAccessView(buf, &ud, &s->out_src_uav))) { + buf->Release(); + LIFT_WARN_ONCE("Leia lift: raw UAV over NeurD's output buffer failed"); + return false; + } + s->out_src_key = res; + } + buf->Release(); + + if (!set_cb(w, h, stride, 0)) { + return false; + } + ID3D11UnorderedAccessView *uavs[2] = {s->out_src_uav, s->out_uav}; + g.nd_ctx->CSSetShader(g.cs_unpack, nullptr, 0); + g.nd_ctx->CSSetConstantBuffers(0, 1, &g.cb); + g.nd_ctx->CSSetUnorderedAccessViews(0, 2, uavs, nullptr); + g.nd_ctx->Dispatch(groups16(w), groups16(h), 1); + unbind_cs(); + return true; +} + +//! Apply one global NeurD property only if it changed. Called with g.mtx held. +bool +prop_i(enum leia_neurd_prop p, int32_t v, int32_t &cache) +{ + if (g.props_valid && cache == v) { + return true; + } + enum leia_neurd_status st = g.nd->set_prop_1i(p, v); + if (st != LEIA_NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD_set_prop_1i(%d, %d) -> %s", (int)p, (int)v, status_str(st)); + cache = -1; + return false; + } + cache = v; + return true; +} + +bool +prop_f(enum leia_neurd_prop p, float v, float &cache) +{ + if (g.props_valid && cache == v) { + return true; + } + enum leia_neurd_status st = g.nd->set_prop_1f(p, v); + if (st != LEIA_NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD_set_prop_1f(%d, %f) -> %s", (int)p, (double)v, status_str(st)); + cache = -1.0f; + return false; + } + cache = v; + return true; +} + +//! Grid for N views: cols = ceil(sqrt(N)), rows = ceil(N/cols), row-major. +void +nview_grid(uint32_t n, uint32_t *cols, uint32_t *rows) +{ + if (n <= 3) { + *cols = n; + *rows = 1; + return; + } + uint32_t c = (uint32_t)std::ceil(std::sqrt((double)n)); + *cols = c; + *rows = (n + c - 1) / c; +} + +int32_t +autoscale_for(const lift_stream *s, int32_t knob_default, uint32_t in_h) +{ + if (!(s->input_scale > 0.0f) || s->input_scale > 1.0f) { + return knob_default; + } + // Smallest NeurD bucket that still covers the requested inference height. + float target = s->input_scale * (float)in_h; + if (target <= 720.0f) { + return LEIA_NEURD_AUTOSCALING_720P; + } + if (target <= 1080.0f) { + return LEIA_NEURD_AUTOSCALING_1080P; + } + if (target <= 1440.0f) { + return LEIA_NEURD_AUTOSCALING_1440P; + } + return LEIA_NEURD_AUTOSCALING_NONE; +} + +} // namespace + +/* + * + * Per-DP handle. + * + */ + +struct leia_lift_neurd +{ + struct knobs k; + bool acquired; //!< Holds a ref on the process instance. + std::vector streams; +}; + +extern "C" void +leia_lift_neurd_map_viewpoint(const float in_m[3], float gain, float out_n[3]) +{ + out_n[0] = clampf(gain * in_m[0] / kIpdRefM, -kViewpointClamp, kViewpointClamp); + out_n[1] = clampf(gain * in_m[1] / kIpdRefM, -kViewpointClamp, kViewpointClamp); + // NeurD's z is a dimensionless depth offset whose relation to viewer + // distance is not specified; head z is deliberately not mapped (see doc). + out_n[2] = 0.0f; +} + +extern "C" struct leia_lift_neurd * +leia_lift_neurd_create(void) +{ + auto *l = new (std::nothrow) leia_lift_neurd(); + if (l == nullptr) { + return nullptr; + } + l->k = read_knobs(); + l->acquired = false; + if (!l->k.enabled) { + LIFT_WARN_ONCE("Leia lift: disabled by DXR_LEIA_LIFT=0"); + } + return l; +} + +extern "C" void +leia_lift_neurd_destroy(struct leia_lift_neurd **plift) +{ + if (plift == nullptr || *plift == nullptr) { + return; + } + struct leia_lift_neurd *l = *plift; + *plift = nullptr; + { + std::lock_guard lock(g.mtx); + for (lift_stream *s : l->streams) { + release_stream_locked(s); + delete s; + g.streams_live.fetch_sub(1); + } + l->streams.clear(); + if (l->acquired) { + l->acquired = false; + if (--g.refcount == 0 && g.state.load() == G_READY && LEIA_NEURD_HAS(g.nd, shrink_memory_pool, 0, 3, 10)) { + // Last user gone: give NeurD's pools back but stay initialised + // (re-init = licence + model load; DPs are recreated on focus change). + g.nd->shrink_memory_pool(); + g.props_valid = false; + } + } + } + delete l; +} + +extern "C" bool +leia_lift_neurd_get_caps(struct leia_lift_neurd *l, struct leia_lift_neurd_caps *out) +{ + if (out == nullptr) { + return false; + } + memset(out, 0, sizeof(*out)); + if (l == nullptr || !l->k.enabled) { + return true; // modes 0 / UNAVAILABLE — a valid answer, not a failure. + } + kick(l->k.backend, l->k.autoscaling); + + uint32_t s = g.state.load(); + out->state = public_state(s); + if (out->state == LEIA_LIFT_STATE_UNAVAILABLE) { + return true; + } + out->modes = LEIA_LIFT_MODE_DEPTH | LEIA_LIFT_MODE_SBS | LEIA_LIFT_MODE_NVIEW; + out->max_streams = kMaxStreams; + out->max_views = kMaxViews; + out->depth_semantics = 0; // relative: per-frame min-max normalised disparity + uint64_t lat = g.latency_ns.load(); + if (s == G_READY) { + std::lock_guard lock(g.mtx); + snprintf(out->backend, sizeof(out->backend), "%s", g.backend_name); + if (lat == 0) { + lat = (g.backend == LEIA_NEURD_BACKEND_DIRECTML) ? kPriorLatencyDirectMlNs + : (g.backend == LEIA_NEURD_BACKEND_CUDA) ? kPriorLatencyCudaNs + : kPriorLatencyOtherNs; + } + } else { + snprintf(out->backend, sizeof(out->backend), "neurd-%s", + l->k.backend == BACKEND_AUTO ? "auto" : backend_str((enum leia_neurd_backend)l->k.backend)); + if (lat == 0) { + lat = kPriorLatencyDirectMlNs; + } + } + out->typical_latency_ns = lat; + return true; +} + +extern "C" bool +leia_lift_neurd_stream_create(struct leia_lift_neurd *l, const struct leia_lift_neurd_stream_desc *desc, uint64_t *out_id) +{ + if (l == nullptr || desc == nullptr || out_id == nullptr || !l->k.enabled) { + return false; + } + if (desc->mode != LEIA_LIFT_MODE_DEPTH && desc->mode != LEIA_LIFT_MODE_SBS && desc->mode != LEIA_LIFT_MODE_NVIEW) { + LIFT_WARN_ONCE("Leia lift: stream_create with unsupported mode %u", desc->mode); + return false; + } + kick(l->k.backend, l->k.autoscaling); + if (public_state(g.state.load()) == LEIA_LIFT_STATE_UNAVAILABLE) { + return false; + } + + std::lock_guard lock(g.mtx); + if (g.streams_live.load() >= kMaxStreams) { + LIFT_WARN_ONCE("Leia lift: stream limit (%u, NeurD's MAX_STREAMS) reached", kMaxStreams); + return false; + } + auto *s = new (std::nothrow) lift_stream(); + if (s == nullptr) { + return false; + } + s->id = g.next_stream_id++; + s->mode = desc->mode; + s->content_hint = desc->content_hint; + s->input_scale = desc->input_scale; + l->streams.push_back(s); + g.streams_live.fetch_add(1); + if (!l->acquired) { + l->acquired = true; + g.refcount++; + } + *out_id = s->id; + U_LOG_W("Leia lift: stream %llu created (mode %u, %s)", (unsigned long long)s->id, s->mode, + s->content_hint == 1 ? "photo" : "video"); + return true; +} + +extern "C" void +leia_lift_neurd_stream_destroy(struct leia_lift_neurd *l, uint64_t id) +{ + if (l == nullptr) { + return; + } + std::lock_guard lock(g.mtx); + for (auto it = l->streams.begin(); it != l->streams.end(); ++it) { + if ((*it)->id == id) { + release_stream_locked(*it); + delete *it; + l->streams.erase(it); + g.streams_live.fetch_sub(1); + return; + } + } +} + +extern "C" bool +leia_lift_neurd_convert(struct leia_lift_neurd *l, + uint64_t id, + void *d3d11_context, + void *input, + uint32_t w, + uint32_t h, + const struct leia_lift_neurd_params *p, + const float *viewpoints_m, + uint32_t viewpoint_floats, + const float *eye_left, + const float *eye_right, + bool eyes_valid, + void **out_resource, + uint32_t *out_w, + uint32_t *out_h, + uint32_t *out_format) +{ + if (l == nullptr || !l->k.enabled || d3d11_context == nullptr || input == nullptr || w == 0 || h == 0 || + out_resource == nullptr || out_w == nullptr || out_h == nullptr || out_format == nullptr) { + return false; + } + *out_resource = nullptr; + + kick(l->k.backend, l->k.autoscaling); + if (g.state.load() != G_READY) { + return false; // ACTIVATING or UNAVAILABLE — caps says which. + } + + LARGE_INTEGER qf, q0, q1; + QueryPerformanceFrequency(&qf); + QueryPerformanceCounter(&q0); + + try { + std::lock_guard lock(g.mtx); + const struct leia_neurd_table *nd = g.nd; + + lift_stream *s = nullptr; + for (lift_stream *it : l->streams) { + if (it->id == id) { + s = it; + break; + } + } + if (s == nullptr) { + LIFT_WARN_ONCE("Leia lift: convert on unknown stream id %llu", (unsigned long long)id); + return false; + } + + // ---- Validate the input. + auto *ctx = static_cast(d3d11_context); + auto *in_res = static_cast(input); + ID3D11Texture2D *in_tex = nullptr; + if (FAILED(in_res->QueryInterface(__uuidof(ID3D11Texture2D), (void **)&in_tex)) || in_tex == nullptr) { + LIFT_WARN_ONCE("Leia lift: input is not an ID3D11Texture2D"); + return false; + } + D3D11_TEXTURE2D_DESC idesc = {}; + in_tex->GetDesc(&idesc); + DXGI_FORMAT family = DXGI_FORMAT_UNKNOWN; + switch (idesc.Format) { + case DXGI_FORMAT_R8G8B8A8_TYPELESS: + case DXGI_FORMAT_R8G8B8A8_UNORM: + case DXGI_FORMAT_R8G8B8A8_UNORM_SRGB: + case DXGI_FORMAT_R8G8B8A8_UINT: family = DXGI_FORMAT_R8G8B8A8_TYPELESS; break; + case DXGI_FORMAT_B8G8R8A8_TYPELESS: + case DXGI_FORMAT_B8G8R8A8_UNORM: + case DXGI_FORMAT_B8G8R8A8_UNORM_SRGB: family = DXGI_FORMAT_B8G8R8A8_TYPELESS; break; + default: break; + } + if (family == DXGI_FORMAT_UNKNOWN || idesc.SampleDesc.Count != 1 || w > idesc.Width || h > idesc.Height) { + in_tex->Release(); + LIFT_WARN_ONCE("Leia lift: unsupported input (DXGI fmt %u, %ux%u, samples %u; asked %ux%u) — need " + "single-sampled RGBA8/BGRA8", + (unsigned)idesc.Format, idesc.Width, idesc.Height, idesc.SampleDesc.Count, w, h); + return false; + } + + // ---- Bind the stream to the caller's device (first use / device change). + ID3D11Device *dev = nullptr; + in_tex->GetDevice(&dev); + if (s->dev != dev) { + release_in_bridge(s); + release_out_bridge(s); + safe_release(s->our_done); + safe_release(s->dev); + s->dev = dev; // keeps GetDevice's reference + D3D11_QUERY_DESC qd = {D3D11_QUERY_EVENT, 0}; + if (FAILED(dev->CreateQuery(&qd, &s->our_done))) { + in_tex->Release(); + LIFT_WARN_ONCE("Leia lift: CreateQuery on the caller's device failed"); + return false; + } + enable_mt_protection(ctx, "caller"); + LUID ours = {}; + if (device_luid(dev, &ours)) { + bool same = ours.HighPart == g.nd_luid.HighPart && ours.LowPart == g.nd_luid.LowPart; + U_LOG_W("Leia lift: stream %llu — caller adapter LUID %08lx:%08lx, NeurD %s (%s); bridging " + "through shared textures either way", + (unsigned long long)s->id, (unsigned long)ours.HighPart, (unsigned long)ours.LowPart, + same ? "on the SAME adapter" : "on a DIFFERENT adapter", + same ? "same-GPU copy" : "cross-adapter copy"); + } + } else { + dev->Release(); + } + + // ---- Lazily create the NeurD stream. + if (s->ns == nullptr) { + enum leia_neurd_status cs = nd->create_stream(&s->ns); + if (cs != LEIA_NEURD_SUCCESS || s->ns == nullptr) { + in_tex->Release(); + s->ns = nullptr; + LIFT_WARN_ONCE("Leia lift: NeurD_create_stream -> %s", status_str(cs)); + return false; + } + } + + if (!ensure_in_bridge(s, w, h, family)) { + in_tex->Release(); + return false; + } + + // ---- 1. Caller's texture -> input bridge (our device), CPU-drained. + D3D11_BOX box = {0, 0, 0, w, h, 1}; + ctx->CopySubresourceRegion(s->in_ours, 0, 0, 0, 0, in_tex, 0, &box); + in_tex->Release(); + if (!drain(ctx, s->our_done, "caller")) { + return false; + } + + // ---- 2. Pack bridge texture -> raw RGBA8 buffer (NeurD device), drained. + if (!set_cb(w, h, w * 4, 0)) { + return false; + } + g.nd_ctx->CSSetShader(g.cs_pack, nullptr, 0); + g.nd_ctx->CSSetConstantBuffers(0, 1, &g.cb); + g.nd_ctx->CSSetShaderResources(0, 1, &s->in_srv); + g.nd_ctx->CSSetUnorderedAccessViews(0, 1, &s->in_uav, nullptr); + g.nd_ctx->Dispatch(groups16(w), groups16(h), 1); + unbind_cs(); + if (!drain(g.nd_ctx, g.nd_done, "NeurD (pack)")) { + return false; + } + + // ---- 3. Global NeurD properties for this stream's mode. + struct leia_lift_neurd_params dp = {-1.0f, 1.0f, 0, 2}; + if (p != nullptr) { + dp = *p; + } + uint32_t views = 2; + uint32_t cols = 2, rows = 1; + int32_t out_type = LEIA_NEURD_OUTPUT_SBS; + if (s->mode == LEIA_LIFT_MODE_DEPTH) { + out_type = LEIA_NEURD_OUTPUT_DEPTH; + views = 1; + cols = rows = 1; + } else if (s->mode == LEIA_LIFT_MODE_NVIEW) { + views = std::min(std::max(dp.view_count, 2u), kMaxViews); + nview_grid(views, &cols, &rows); + } + const bool auto_conv = !(dp.convergence >= 0.0f); + const float gain = (dp.strength > 0.0f) ? clampf(dp.strength, 0.1f, 10.0f) : 1.0f; + const int32_t inpaint = (dp.inpaint == 1) ? LEIA_NEURD_INPAINT_V1_BLUR : LEIA_NEURD_INPAINT_V1_STRETCH; + + bool props_ok = prop_i(LEIA_NEURD_PROP_OUTPUT_TYPE, out_type, g.p_out_type) && + prop_i(LEIA_NEURD_PROP_OUTPUT_TILES_W, (int32_t)cols, g.p_tiles_w) && + prop_i(LEIA_NEURD_PROP_OUTPUT_TILES_H, (int32_t)rows, g.p_tiles_h) && + prop_i(LEIA_NEURD_PROP_INPAINT_TYPE, inpaint, g.p_inpaint) && + prop_i(LEIA_NEURD_PROP_INPUT_AUTOSCALING, autoscale_for(s, l->k.autoscaling, h), + g.p_autoscale) && + prop_i(LEIA_NEURD_PROP_AUTO_CONVERGENCE, auto_conv ? 1 : 0, g.p_autoconv) && + prop_f(LEIA_NEURD_PROP_GAIN_MULTIPLIER, gain, g.p_gain); + if (props_ok && !auto_conv) { + props_ok = prop_f(LEIA_NEURD_PROP_CONVERGENCE, clampf(dp.convergence, -0.2f, 0.2f), g.p_conv); + } + g.props_valid = props_ok; + if (!props_ok) { + return false; + } + + // ---- 4. Viewpoints (SBS / NVIEW): explicit, else tracked eyes, else default. + std::vector vp; + if (s->mode != LEIA_LIFT_MODE_DEPTH) { + const uint32_t slots = cols * rows; // NeurD wants one triplet per grid tile + std::vector src_m; // metres, (x,y,z) per view + if (viewpoints_m != nullptr && viewpoint_floats >= 3 && viewpoint_floats % 3 == 0) { + const uint32_t k = viewpoint_floats / 3; + if (k == views) { + src_m.assign(viewpoints_m, viewpoints_m + viewpoint_floats); + } else if (k == 2) { + eye_left = viewpoints_m; + eye_right = viewpoints_m + 3; + eyes_valid = true; + } else { + LIFT_WARN_ONCE("Leia lift: %u explicit viewpoints for %u views — using tracked eyes", k, + views); + } + } + if (src_m.empty() && eyes_valid && eye_left != nullptr && eye_right != nullptr) { + // N views one eye-baseline apart, centred on the eye midpoint. + float c[3], d[3]; + for (int a = 0; a < 3; a++) { + c[a] = 0.5f * (eye_left[a] + eye_right[a]); + d[a] = eye_right[a] - eye_left[a]; + } + for (uint32_t i = 0; i < views; i++) { + float t = (float)i - 0.5f * (float)(views - 1); + for (int a = 0; a < 3; a++) { + src_m.push_back(c[a] + t * d[a]); + } + } + } + if (!src_m.empty()) { + vp.reserve(slots * 3); + for (uint32_t i = 0; i < slots; i++) { + uint32_t v = std::min(i, views - 1); // extra grid slots repeat the last view + float n[3]; + leia_lift_neurd_map_viewpoint(&src_m[v * 3], l->k.view_gain, n); + vp.insert(vp.end(), n, n + 3); + } + } else if (s->mode == LEIA_LIFT_MODE_NVIEW) { + // Untracked N-view: evenly spaced, one unit apart (the ±0.5 stereo + // spacing extended), centred on 0. + for (uint32_t i = 0; i < slots; i++) { + uint32_t v = std::min(i, views - 1); + vp.push_back((float)v - 0.5f * (float)(views - 1)); + vp.push_back(0.0f); + vp.push_back(0.0f); + } + } + // SBS untracked: vp stays empty -> NeurD's own default ±0.5 pattern. + } + + // ---- 5. Convert (blocking). + struct leia_neurd_image nin = {}; + nin.data = s->in_buf; + nin.width = (int32_t)w; + nin.height = (int32_t)h; + nin.stride = (int32_t)(w * 4); + nin.pix_fmt = LEIA_NEURD_PIXEL_FORMAT_RGBA8; + struct leia_neurd_image nout = {}; + enum leia_neurd_status st; + if (!vp.empty() && !g.interactive_unavailable) { + st = nd->convert_stream_dx_interactive(s->ns, &nin, vp.data(), (int)vp.size(), + LEIA_NEURD_PIXEL_FORMAT_RGBA8, &nout); + if (st == LEIA_NEURD_UNAVAILABLE_OUTDATED_RUNTIME) { + g.interactive_unavailable = true; + U_LOG_W("Leia lift: NeurD interactive convert unavailable — tracked viewpoints ignored"); + } + if (st != LEIA_NEURD_SUCCESS && s->mode == LEIA_LIFT_MODE_SBS) { + LIFT_WARN_ONCE("Leia lift: interactive convert -> %s; retrying with the default pattern", + status_str(st)); + st = nd->convert_stream_dx(s->ns, &nin, LEIA_NEURD_PIXEL_FORMAT_RGBA8, &nout); + } + } else if (s->mode == LEIA_LIFT_MODE_NVIEW && g.interactive_unavailable) { + LIFT_WARN_ONCE("Leia lift: N-view needs NeurD >= 0.4.5 (interactive convert)"); + return false; + } else { + st = nd->convert_stream_dx(s->ns, &nin, LEIA_NEURD_PIXEL_FORMAT_RGBA8, &nout); + } + if (st != LEIA_NEURD_SUCCESS || nout.data == nullptr) { + LIFT_WARN_ONCE("Leia lift: NeurD convert failed: %s", status_str(st)); + return false; + } + + // ---- 6. Output -> bridge (NeurD device), drained; hand ours back. + if (!stage_output(s, nout) || !drain(g.nd_ctx, g.nd_done, "NeurD (unpack)")) { + return false; + } + + *out_resource = static_cast(s->out_ours); + *out_w = s->out_w; + *out_h = s->out_h; + *out_format = (uint32_t)DXGI_FORMAT_R8G8B8A8_UNORM; + } catch (...) { + // std::vector / std::mutex can throw; nothing may escape into the C runtime. + LIFT_WARN_ONCE("Leia lift: exception in convert — frame dropped"); + return false; + } + + QueryPerformanceCounter(&q1); + const uint64_t ns = (uint64_t)((double)(q1.QuadPart - q0.QuadPart) * 1e9 / (double)qf.QuadPart); + const uint64_t prev = g.latency_ns.load(); + g.latency_ns.store(prev == 0 ? ns : (prev * 7 + ns) / 8); + return true; +} diff --git a/src/drv_leia/leia_lift_neurd.h b/src/drv_leia/leia_lift_neurd.h new file mode 100644 index 0000000..19dc65c --- /dev/null +++ b/src/drv_leia/leia_lift_neurd.h @@ -0,0 +1,151 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: Apache-2.0 +/*! + * @file + * @brief NeurD-backed 2D->3D conversion ("lift") for the Leia D3D11 display + * processor. See docs/lift-neurd.md. + * + * Plug-in-owned types only — the D3D11 DP translates the runtime's + * xrt_dp_lift_* contract (behind XRT_DP_D3D11_HAS_LIFT) onto this, so the + * module builds against any runtime pin. + * + * Process model: ONE NeurD instance per process, loaded + initialised lazily on + * a background thread the first time a DP asks for caps or a stream (never at + * DP create, never on the caller's thread — first activation runs a licence + * check over the network). Each DP owns a leia_lift_neurd handle holding its + * streams; handles ref-count the instance. + * + * Absent NeurD.dll (or DXR_LEIA_LIFT=0): caps report modes=0 / state=0 and + * every other entry point returns false. Nothing is loaded. + * + * Threading: every entry point is callable from any thread. convert() is + * synchronous and blocking (tens of ms) — call it only from a worker thread, + * never a render/IPC thread. Calls are serialised process-wide (NeurD's + * properties are global). + * + * @ingroup drv_leia + */ + +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//! Mode bits (identical values to the runtime's lift contract). +#define LEIA_LIFT_MODE_DEPTH 1u +#define LEIA_LIFT_MODE_SBS 2u +#define LEIA_LIFT_MODE_NVIEW 4u + +//! Caps state (identical values to the runtime's lift contract). +#define LEIA_LIFT_STATE_UNAVAILABLE 0u +#define LEIA_LIFT_STATE_ACTIVATING 1u +#define LEIA_LIFT_STATE_READY 2u + +struct leia_lift_neurd_caps +{ + uint32_t modes; //!< LEIA_LIFT_MODE_* bits; 0 when unavailable. + uint32_t max_streams; //!< Concurrent streams (NeurD limit, process-wide). + uint32_t max_views; //!< Max view_count for NVIEW. + uint32_t depth_semantics; //!< 0 = relative (NeurD: min-max normalised disparity). + uint32_t state; //!< LEIA_LIFT_STATE_*. + uint64_t typical_latency_ns; + char backend[32]; //!< e.g. "neurd-directml"; "" when unavailable. +}; + +struct leia_lift_neurd_stream_desc +{ + uint32_t mode; //!< One LEIA_LIFT_MODE_* value. + uint32_t content_hint; //!< 0 video, 1 photo (advisory; the DX path is video-model only). + float input_scale; //!< (0,1] fraction of input height to infer at; <=0 = knob default. +}; + +struct leia_lift_neurd_params +{ + float convergence; //!< <0 = NeurD auto-convergence; else NeurD units, clamped to [-0.2, 0.2]. + float strength; //!< NeurD gain multiplier; <=0 = 1.0; clamped to [0.1, 10]. + uint32_t inpaint; //!< 0 stretch, 1 blur. + uint32_t view_count; //!< NVIEW only (2..max_views). +}; + +struct leia_lift_neurd; + +/*! + * Create a per-DP handle. Reads the env knobs (DXR_LEIA_LIFT, + * DXR_LEIA_LIFT_BACKEND, DXR_LEIA_LIFT_SCALE, DXR_LEIA_LIFT_VIEW_GAIN) once. + * Cheap: no DLL load, no thread. Never fails (a disabled handle reports + * unavailable). + */ +struct leia_lift_neurd * +leia_lift_neurd_create(void); + +void +leia_lift_neurd_destroy(struct leia_lift_neurd **lift); + +//! Non-blocking. May kick the background load/activation (or a licence retry). +bool +leia_lift_neurd_get_caps(struct leia_lift_neurd *lift, struct leia_lift_neurd_caps *out); + +//! Non-blocking. Succeeds while NeurD is still activating (the NeurD stream is +//! created lazily on the first convert); fails when unavailable / out of slots. +bool +leia_lift_neurd_stream_create(struct leia_lift_neurd *lift, + const struct leia_lift_neurd_stream_desc *desc, + uint64_t *out_id); + +void +leia_lift_neurd_stream_destroy(struct leia_lift_neurd *lift, uint64_t id); + +/*! + * Convert one frame. SYNCHRONOUS and blocking. + * + * @param d3d11_context Caller's immediate context (the device that owns + * @p input). Must be ID3D11Multithread-protected — it is + * enabled here if it is not. + * @param input ID3D11Texture2D* (as ID3D11Resource*), RGBA8 or BGRA8 + * family, single-sampled; the top-left w x h is used. + * @param viewpoints_m Optional explicit viewpoints: (x,y,z) triplets in + * METRES, display space (same space as tracked eyes). + * NULL/0 = use @p eye_left / @p eye_right. + * @param eyes_valid False (or NULL eyes) = no tracked viewer: NeurD's + * default pattern is used. + * @param out_resource ID3D11Texture2D* on the caller's device, owned by the + * stream, valid until the next convert on this stream. + * @param out_format DXGI_FORMAT (always R8G8B8A8_UNORM today). + */ +bool +leia_lift_neurd_convert(struct leia_lift_neurd *lift, + uint64_t id, + void *d3d11_context, + void *input, + uint32_t w, + uint32_t h, + const struct leia_lift_neurd_params *params, + const float *viewpoints_m, + uint32_t viewpoint_floats, + const float *eye_left, + const float *eye_right, + bool eyes_valid, + void **out_resource, + uint32_t *out_w, + uint32_t *out_h, + uint32_t *out_format); + +/*! + * Pure helper, exposed for tests + docs: map display-space positions (metres) + * to NeurD's dimensionless viewpoint units. See docs/lift-neurd.md. + * + * x_n = clamp(gain * x_m / 0.063), y_n = clamp(gain * y_m / 0.063), z_n = 0. + * A centred viewer at 63 mm IPD maps to x = -0.5 / +0.5 — exactly NeurD's + * default stereo pattern — so gain 1.0 reproduces the untracked output and + * head motion becomes look-around. + */ +void +leia_lift_neurd_map_viewpoint(const float in_m[3], float gain, float out_n[3]); + +#ifdef __cplusplus +} +#endif diff --git a/src/drv_leia/leia_neurd_abi.h b/src/drv_leia/leia_neurd_abi.h new file mode 100644 index 0000000..02d4056 --- /dev/null +++ b/src/drv_leia/leia_neurd_abi.h @@ -0,0 +1,223 @@ +// Copyright 2026, Leia Inc. +// SPDX-License-Identifier: Apache-2.0 +/*! + * @file + * @brief Minimal binary-interface declarations for the NeurD runtime DLL + * (Leia's 2D->3D conversion library), used by leia_lift_neurd.cpp. + * + * NeurD ships as a DLL whose only exported symbol is `NeurD_load`; it returns a + * pointer to a versioned table of function pointers. This header declares just + * enough of that contract to drive the D3D11 stream path — the enums and + * structs the plug-in passes by value, and the table layout. It is written + * against NeurD 0.4.6 and contains no NeurD code. + * + * ABI rules this relies on (NeurD's own compatibility policy): + * - table entries are append-only, never reordered or re-signatured; + * - `version` in the table is the LOADED runtime's version, so an entry may + * only be read when `version >= the entry's introduction version` (an older + * runtime's table is physically shorter — reading past it is UB). Use + * LEIA_NEURD_HAS() for every call. + * + * If NeurD ever breaks this (it would be a NeurD major), the version check in + * leia_lift_neurd.cpp rejects a major other than 0 and lift reports unavailable. + * + * @ingroup drv_leia + */ + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_WIN32) +#define LEIA_NEURD_CALL __stdcall +#else +#define LEIA_NEURD_CALL +#endif + +#define LEIA_NEURD_MAKE_VERSION(major, minor, patch) \ + ((((uint64_t)(major)&0xffffULL) << 48) | (((uint64_t)(minor)&0xffffULL) << 32) | \ + ((uint64_t)(patch)&0xffffffffULL)) +#define LEIA_NEURD_VERSION_MAJOR(v) ((uint32_t)(((uint64_t)(v) >> 48) & 0xffffULL)) +#define LEIA_NEURD_VERSION_MINOR(v) ((uint32_t)(((uint64_t)(v) >> 32) & 0xffffULL)) +#define LEIA_NEURD_VERSION_PATCH(v) ((uint32_t)((uint64_t)(v)&0xffffffffULL)) + +//! The table revision this header describes (requested in NeurD_load). +#define LEIA_NEURD_BUILT_AGAINST LEIA_NEURD_MAKE_VERSION(0, 4, 6) + +enum leia_neurd_status +{ + LEIA_NEURD_SUCCESS = 0, + LEIA_NEURD_IN_PROGRESS = 1, + LEIA_NEURD_GENERIC_ERROR = -1, + LEIA_NEURD_INVALID_ARG = -2, + LEIA_NEURD_INVALID_LICENSE = -3, + LEIA_NEURD_UNAVAILABLE_OUTDATED_RUNTIME = -4, + LEIA_NEURD_NOT_INITIALIZED = -5, + LEIA_NEURD_LICENSE_NETWORK_ERROR = -6, + LEIA_NEURD_INVALID_MODEL = -7, + LEIA_NEURD_FILESYSTEM_ERROR = -8, + LEIA_NEURD_NETWORK_ERROR = -9, + LEIA_NEURD_ABORTED = -10, + LEIA_NEURD_STATUS_MAKE_32BIT = 0x7FFFFFFF +}; + +enum leia_neurd_pixel_format +{ + LEIA_NEURD_PIXEL_FORMAT_NV12 = 0, + LEIA_NEURD_PIXEL_FORMAT_BGR8 = 1, + LEIA_NEURD_PIXEL_FORMAT_BGR32 = 2, + LEIA_NEURD_PIXEL_FORMAT_RGBA8 = 3, + LEIA_NEURD_PIXEL_FORMAT_RGB8 = 4, + LEIA_NEURD_PIXEL_FORMAT_MAKE_32BIT = 0x7FFFFFFF +}; + +//! For the DX entry points `data` is an ID3D11Buffer* (input) and an +//! ID3D11Buffer* or ID3D11Texture2D* (output, owned by NeurD). +struct leia_neurd_image +{ + void *data; + int32_t width; + int32_t height; + int32_t stride; + enum leia_neurd_pixel_format pix_fmt; +}; + +enum leia_neurd_output_type +{ + LEIA_NEURD_OUTPUT_SBS = 0, + LEIA_NEURD_OUTPUT_TB = 1, + LEIA_NEURD_OUTPUT_DEPTH = 2, +}; + +enum leia_neurd_autoscaling +{ + LEIA_NEURD_AUTOSCALING_NONE = 0, + LEIA_NEURD_AUTOSCALING_720P = 1, + LEIA_NEURD_AUTOSCALING_1080P = 2, + LEIA_NEURD_AUTOSCALING_1440P = 3, +}; + +enum leia_neurd_inpaint +{ + LEIA_NEURD_INPAINT_V1_STRETCH = 0, + LEIA_NEURD_INPAINT_V1_BLUR = 1, +}; + +enum leia_neurd_model_id +{ + LEIA_NEURD_MODEL_PHOTO_RELATIVE_QUALITY = 0, + LEIA_NEURD_MODEL_VIDEO_RELATIVE_FAST = 1, + LEIA_NEURD_MODEL_VIDEO_METRIC_QUALITY = 2, +}; + +struct leia_neurd_init_options +{ + uint32_t struct_size; + int32_t photo_model_id; + int32_t video_model_id; +}; + +//! Property ids (NeurD_prop). All are process-global inside NeurD. +enum leia_neurd_prop +{ + LEIA_NEURD_PROP_OUTPUT_TYPE = 0, //!< int, leia_neurd_output_type + LEIA_NEURD_PROP_GAIN_MULTIPLIER = 2, //!< float, default 1 + LEIA_NEURD_PROP_CONVERGENCE = 3, //!< float, must be in [-0.2, 0.2] + LEIA_NEURD_PROP_INPUT_AUTOSCALING = 7, //!< int, leia_neurd_autoscaling (default 1440p) + LEIA_NEURD_PROP_INPAINT_TYPE = 8, //!< int, leia_neurd_inpaint + LEIA_NEURD_PROP_OUTPUT_TILES_W = 9, //!< int, default 1 + LEIA_NEURD_PROP_OUTPUT_TILES_H = 10, //!< int, default 1 + LEIA_NEURD_PROP_AUTO_CONVERGENCE = 11, //!< int bool, default true + LEIA_NEURD_PROP_MAKE_32BIT = 0x7FFFFFFF +}; + +enum leia_neurd_backend +{ + LEIA_NEURD_BACKEND_CUDA = 0, + LEIA_NEURD_BACKEND_DIRECTML = 1, + LEIA_NEURD_BACKEND_OPENVINO = 2, + LEIA_NEURD_BACKEND_MAKE_32BIT = 0x7FFFFFFF +}; + +struct leia_neurd_load_request +{ + uint64_t version; +}; + +struct leia_neurd_stream; // opaque + +typedef void *leia_neurd_fn_unused; + +/*! + * The NeurD function table, in NeurD's table order (append-only). Entries the + * plug-in never calls are typed as leia_neurd_fn_unused — they only hold their + * slot. The trailing comment is the version that introduced each entry. + */ +struct leia_neurd_table +{ + uint64_t version; + + enum leia_neurd_status(LEIA_NEURD_CALL *set_prop_1f)(enum leia_neurd_prop prop, float value); // 0.2.1 + enum leia_neurd_status(LEIA_NEURD_CALL *set_prop_1i)(enum leia_neurd_prop prop, int32_t value); // 0.2.1 + leia_neurd_fn_unused deprecated_init_license; // 0.2.1 + void(LEIA_NEURD_CALL *deinit)(void); // 0.2.1 + leia_neurd_fn_unused convert_cuda; // 0.2.1 + enum leia_neurd_status(LEIA_NEURD_CALL *init)(int32_t multithreaded); // 0.2.2 + leia_neurd_fn_unused get_prop_1f; // 0.2.2 + leia_neurd_fn_unused get_prop_1i; // 0.2.2 + enum leia_neurd_status(LEIA_NEURD_CALL *set_logger_callback)(void (*cb)(const char *msg, int size)); // 0.2.5 + leia_neurd_fn_unused get_cuda_runtime_version; // 0.2.7 + leia_neurd_fn_unused read_multiview; // 0.3.2 + leia_neurd_fn_unused get_multiview_attributes; // 0.3.2 + leia_neurd_fn_unused render_multiview; // 0.3.2 + leia_neurd_fn_unused render_multiview_interactive_cpu; // 0.3.2 + leia_neurd_fn_unused render_multiview_interactive_cuda; // 0.3.2 + leia_neurd_fn_unused free_multiview; // 0.3.2 + enum leia_neurd_status(LEIA_NEURD_CALL *set_backend)(enum leia_neurd_backend backend); // 0.3.4 + enum leia_neurd_status(LEIA_NEURD_CALL *get_backend)(enum leia_neurd_backend *backend); // 0.3.4 + leia_neurd_fn_unused convert_cpu; // 0.3.5 + leia_neurd_fn_unused prepare_multiview; // 0.3.7 + leia_neurd_fn_unused convert_dx; // 0.3.8 + void(LEIA_NEURD_CALL *shrink_memory_pool)(void); // 0.3.10 + enum leia_neurd_status(LEIA_NEURD_CALL *create_stream)(struct leia_neurd_stream **stream); // 0.3.11 + enum leia_neurd_status(LEIA_NEURD_CALL *destroy_stream)(struct leia_neurd_stream *stream); // 0.3.11 + leia_neurd_fn_unused convert_stream_cpu; // 0.3.11 + leia_neurd_fn_unused convert_stream_cuda; // 0.3.11 + enum leia_neurd_status(LEIA_NEURD_CALL *convert_stream_dx)(struct leia_neurd_stream *stream, + const struct leia_neurd_image *input, + enum leia_neurd_pixel_format output_pix_fmt, + struct leia_neurd_image *output); // 0.3.11 + leia_neurd_fn_unused convert_batch_cpu; // 0.4.3 + leia_neurd_fn_unused convert_batch_cuda; // 0.4.3 + leia_neurd_fn_unused convert_batch_dx; // 0.4.3 + void *(LEIA_NEURD_CALL *get_dx_device)(void); // 0.4.3 + leia_neurd_fn_unused convert_stream_cuda_interactive; // 0.4.5 + enum leia_neurd_status(LEIA_NEURD_CALL *convert_stream_dx_interactive)( + struct leia_neurd_stream *stream, + const struct leia_neurd_image *input, + const float *viewpoint_xyz, + int size, + enum leia_neurd_pixel_format output_pix_fmt, + struct leia_neurd_image *output); // 0.4.5 + leia_neurd_fn_unused convert_batch_cuda_interactive; // 0.4.5 + leia_neurd_fn_unused convert_batch_dx_interactive; // 0.4.5 + enum leia_neurd_status(LEIA_NEURD_CALL *init_with_options)(int32_t multithreaded, + const struct leia_neurd_init_options *opts); // 0.4.6 +}; + +typedef const struct leia_neurd_table *(LEIA_NEURD_CALL *leia_neurd_load_fn)(const struct leia_neurd_load_request *req); + +/*! + * True when the loaded table carries @p field. The version test comes FIRST and + * short-circuits, so a shorter (older) table is never read past its end. + */ +#define LEIA_NEURD_HAS(tbl, field, maj, min, pat) \ + ((tbl) != NULL && (tbl)->version >= LEIA_NEURD_MAKE_VERSION(maj, min, pat) && (tbl)->field != NULL) + +#ifdef __cplusplus +} +#endif From 3f6f00e824363bbdc059f7705c6e945436e120fe Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 13:17:54 -0700 Subject: [PATCH 02/21] feat(lift): wire the four NeurD lift slots into the D3D11 DP lift_get_caps / lift_stream_create / lift_stream_destroy / lift_convert, compiled only when the runtime headers announce XRT_DP_D3D11_HAS_LIFT (runtime feat/lift-ext); struct_size-bounded reads of every versioned struct. Tracked eyes come from the existing SR eye path. The lift handle is created at DP create (reads DXR_LEIA_LIFT* once, loads nothing) and destroyed before the weaver. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/drv_leia/CMakeLists.txt | 4 + src/drv_leia/leia_display_processor_d3d11.cpp | 144 ++++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/src/drv_leia/CMakeLists.txt b/src/drv_leia/CMakeLists.txt index b8348d3..7147563 100644 --- a/src/drv_leia/CMakeLists.txt +++ b/src/drv_leia/CMakeLists.txt @@ -183,6 +183,10 @@ if(EXISTS "${SR_PATH}/lib/SimulatedRealityDirectX.lib") list(APPEND LEIA_SOURCES leia_sr_d3d11.cpp leia_sr_d3d11.h leia_display_processor_d3d11.cpp leia_display_processor_d3d11.h + # 2D->3D lift via NeurD (docs/lift-neurd.md). NeurD.dll is loaded + # dynamically at first use — no NeurD headers, libs or DLLs are needed + # to build, and a box without NeurD loads the plug-in unchanged. + leia_lift_neurd.cpp leia_lift_neurd.h leia_neurd_abi.h leia_sr_d3d12.cpp leia_sr_d3d12.h leia_display_processor_d3d12.cpp leia_display_processor_d3d12.h) set(LEIA_SR_D3D11_AVAILABLE TRUE) diff --git a/src/drv_leia/leia_display_processor_d3d11.cpp b/src/drv_leia/leia_display_processor_d3d11.cpp index 0054843..5cddc70 100644 --- a/src/drv_leia/leia_display_processor_d3d11.cpp +++ b/src/drv_leia/leia_display_processor_d3d11.cpp @@ -52,6 +52,20 @@ #define DXR_LEIA_DP_D3D11_BACKEND_STATE 1 #endif +/* + * 2D->3D lift (runtime feat/lift-ext, XRT_DP_D3D11_HAS_LIFT) — four appended + * slots backed by NeurD (leia_lift_neurd.cpp, docs/lift-neurd.md). Same + * coupled-addition pattern: against older runtime headers the slots compile out + * and struct_size tells the runtime they are absent. The NeurD module itself is + * runtime-header-independent and always built; it never loads NeurD.dll until + * a lift slot is actually called. + */ +#if defined(XRT_DP_D3D11_HAS_LIFT) +#define DXR_LEIA_DP_D3D11_LIFT 1 +#include "leia_lift_neurd.h" +#include // offsetof +#endif + // Fullscreen quad vertex shader (4 vertices, triangle strip via SV_VertexID) static const char *blit_vs_source = R"( @@ -513,6 +527,12 @@ struct leia_display_processor_d3d11_impl //! it over the live desktop. NULL ⟹ no backdrop (desktop-only). ID3D11ShaderResourceView *backdrop_srv; //!< NOT owned (compositor-owned). uint32_t backdrop_w, backdrop_h; + +#ifdef DXR_LEIA_DP_D3D11_LIFT + //! NeurD 2D->3D lift handle (owned; created at factory time — cheap, no + //! DLL load — destroyed in leia_dp_d3d11_destroy). NULL only on OOM. + struct leia_lift_neurd *lift; +#endif }; static inline struct leia_display_processor_d3d11_impl * @@ -2287,6 +2307,114 @@ leia_dp_d3d11_set_window(struct xrt_display_processor_d3d11 *xdp, void *window_h } #endif +#ifdef DXR_LEIA_DP_D3D11_LIFT +/* + * 2D->3D lift slots. Thin translation of the runtime's xrt_dp_lift_* contract + * onto leia_lift_neurd; every size-versioned struct is read only as far as its + * struct_size covers (ADR-020 append-only rule), so a runtime that grows these + * structs keeps working against this plug-in and vice versa. + */ +#define LEIA_LIFT_COVERS(ptr, type, field) \ + ((ptr)->struct_size >= offsetof(type, field) + sizeof(((type *)0)->field)) + +static bool +leia_dp_d3d11_lift_get_caps(struct xrt_display_processor_d3d11 *xdp, struct xrt_dp_lift_caps *out) +{ + struct leia_display_processor_d3d11_impl *ldp = leia_dp_d3d11(xdp); + if (out == NULL || out->struct_size < offsetof(struct xrt_dp_lift_caps, state) + sizeof(uint32_t)) { + return false; + } + struct leia_lift_neurd_caps c = {}; + if (!leia_lift_neurd_get_caps(ldp->lift, &c)) { + return false; + } + struct xrt_dp_lift_caps full = {}; + full.struct_size = out->struct_size; + full.modes = c.modes; + full.max_streams = c.max_streams; + full.max_views = c.max_views; + full.depth_semantics = c.depth_semantics; + full.state = c.state; + full.typical_latency_ns = c.typical_latency_ns; + static_assert(sizeof(full.backend) <= sizeof(c.backend), "backend name buffers"); + memcpy(full.backend, c.backend, sizeof(full.backend)); + full.backend[sizeof(full.backend) - 1] = '\0'; + memcpy(out, &full, out->struct_size < sizeof(full) ? out->struct_size : sizeof(full)); + return true; +} + +static bool +leia_dp_d3d11_lift_stream_create(struct xrt_display_processor_d3d11 *xdp, + const struct xrt_dp_lift_stream_info *info, + uint64_t *out_id) +{ + struct leia_display_processor_d3d11_impl *ldp = leia_dp_d3d11(xdp); + if (info == NULL || out_id == NULL || !LEIA_LIFT_COVERS(info, struct xrt_dp_lift_stream_info, mode)) { + return false; + } + struct leia_lift_neurd_stream_desc d = {}; + d.mode = info->mode; + d.content_hint = LEIA_LIFT_COVERS(info, struct xrt_dp_lift_stream_info, content_hint) ? info->content_hint : 0; + d.input_scale = LEIA_LIFT_COVERS(info, struct xrt_dp_lift_stream_info, input_scale) ? info->input_scale : 0.0f; + return leia_lift_neurd_stream_create(ldp->lift, &d, out_id); +} + +static void +leia_dp_d3d11_lift_stream_destroy(struct xrt_display_processor_d3d11 *xdp, uint64_t id) +{ + leia_lift_neurd_stream_destroy(leia_dp_d3d11(xdp)->lift, id); +} + +static bool +leia_dp_d3d11_lift_convert(struct xrt_display_processor_d3d11 *xdp, + uint64_t id, + void *d3d11_context, + void *input_resource, + uint32_t w, + uint32_t h, + const struct xrt_dp_lift_params *p, + const float *viewpoints_xyz, + uint32_t viewpoint_floats, + void **out_resource, + uint32_t *out_w, + uint32_t *out_h, + uint32_t *out_format) +{ + struct leia_display_processor_d3d11_impl *ldp = leia_dp_d3d11(xdp); + + struct leia_lift_neurd_params lp = {-1.0f, 1.0f, 0, 2}; + if (p != NULL) { + if (LEIA_LIFT_COVERS(p, struct xrt_dp_lift_params, convergence)) { + lp.convergence = p->convergence; + } + if (LEIA_LIFT_COVERS(p, struct xrt_dp_lift_params, strength)) { + lp.strength = p->strength; + } + if (LEIA_LIFT_COVERS(p, struct xrt_dp_lift_params, inpaint)) { + lp.inpaint = p->inpaint; + } + if (LEIA_LIFT_COVERS(p, struct xrt_dp_lift_params, view_count)) { + lp.view_count = p->view_count; + } + } + + // Tracked eyes (metres, display space) — the same source the runtime's + // get_predicted_eye_positions slot reads; the wrapper is exception-safe. + // Only consulted when the runtime did not pass explicit viewpoints. + float left[3] = {0, 0, 0}, right[3] = {0, 0, 0}; + bool eyes_valid = false; + if ((viewpoints_xyz == NULL || viewpoint_floats == 0) && ldp->leiasr != NULL && + leiasr_d3d11_get_predicted_eye_positions(ldp->leiasr, left, right)) { + const float dx = right[0] - left[0], dy = right[1] - left[1], dz = right[2] - left[2]; + eyes_valid = (dx * dx + dy * dy + dz * dz) > 1e-6f; // same tracking-loss test as the eye slot + } + + return leia_lift_neurd_convert(ldp->lift, id, d3d11_context, input_resource, w, h, &lp, + (viewpoint_floats > 0) ? viewpoints_xyz : NULL, viewpoint_floats, left, right, + eyes_valid, out_resource, out_w, out_h, out_format); +} +#endif // DXR_LEIA_DP_D3D11_LIFT + static void leia_dp_d3d11_destroy(struct xrt_display_processor_d3d11 *xdp) { @@ -2305,6 +2433,12 @@ leia_dp_d3d11_destroy(struct xrt_display_processor_d3d11 *xdp) ldp->blit_cb->Release(); } +#ifdef DXR_LEIA_DP_D3D11_LIFT + // Before the weaver: lift streams hold bridge resources on the service + // device; the NeurD instance itself stays up for the process (see module). + leia_lift_neurd_destroy(&ldp->lift); +#endif + compose_release_resources(ldp); ck_release_resources(ldp); @@ -2416,6 +2550,16 @@ leia_dp_d3d11_init_vtable(struct leia_display_processor_d3d11_impl *ldp) #ifdef DXR_LEIA_DP_D3D11_BACKEND_STATE ldp->base.get_backend_state = leia_dp_d3d11_get_backend_state; // #158 SR restart self-heal (slot 21) #endif +#ifdef DXR_LEIA_DP_D3D11_LIFT + ldp->base.lift_get_caps = leia_dp_d3d11_lift_get_caps; // 2D->3D lift (NeurD) + ldp->base.lift_stream_create = leia_dp_d3d11_lift_stream_create; + ldp->base.lift_stream_destroy = leia_dp_d3d11_lift_stream_destroy; + ldp->base.lift_convert = leia_dp_d3d11_lift_convert; + ldp->lift = leia_lift_neurd_create(); // reads DXR_LEIA_LIFT* once; loads nothing + U_LOG_W("Leia D3D11 DP: lift slots WIRED (NeurD, loaded on first use)"); +#else + U_LOG_W("Leia D3D11 DP: lift slots NOT COMPILED (runtime headers predate XRT_DP_D3D11_HAS_LIFT)"); +#endif } From 5bf154448ad730798d3b268614c1db7e8f1ca034 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 13:19:03 -0700 Subject: [PATCH 03/21] docs(lift): NeurD lift architecture, threading, licensing, bridge, knobs, verification Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 1 + docs/README.md | 2 + docs/lift-neurd.md | 226 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 229 insertions(+) create mode 100644 docs/lift-neurd.md diff --git a/CLAUDE.md b/CLAUDE.md index 669e5d9..244c6db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,6 +98,7 @@ ADR-020 spec: [`displayxr-runtime/docs/adr/ADR-020-plugin-abi-policy.md`](https: | `src/drv_leia/leia_display_processor*.{cpp,h}` | Per-API display processors (D3D11, D3D12, GL). Implement `xrt_display_processor_` vtables; the runtime's compositor invokes them per-frame. | | `src/drv_leia/leia_sr_*.{cpp,h}` | SR SDK weaver wrappers. The SDK throws `std::runtime_error` as routine internal control flow (~11/frame in some paths) — every call wrapped in try/catch. | | `src/drv_leia/leia_bg_capture_win.{cpp,h}` | WGC background capture for compose-under transparency (Leia transparency model). | +| `src/drv_leia/leia_lift_neurd.{cpp,h}`, `leia_neurd_abi.h` | 2D→3D lift slots on the D3D11 DP, backed by a dynamically loaded NeurD.dll (absent ⇒ caps unavailable, nothing loaded). Slots compile only against runtime headers with `XRT_DP_D3D11_HAS_LIFT`. See [`docs/lift-neurd.md`](docs/lift-neurd.md). | | `src/drv_leia/leia_edid_probe.c` | EDID-based hardware detection — answers "is a Leia display attached?" before the SR SDK initializes. | | `src/drv_leia_linux/leia_sr_linux.h` | **Linux weaver-backend seam** — interface shaped 1:1 by `docs/leia-linux-sdk-contract.md` (every declaration cites its R-* requirement). Track B implements it against the real SDK. | | `src/drv_leia_linux/leia_sr_stub.c` | Track A stub backend: canned panel info + passthrough SBS blit, `TODO(Track B)` at every body. | diff --git a/docs/README.md b/docs/README.md index 38b3c0b..213fb2d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,7 @@ Leia is the first 3D-display vendor integrated into DisplayXR. This directory do | Display processor (per API) | `leia_display_processor.{cpp,h}` (base), `leia_display_processor_d3d11.{cpp,h}`, `leia_display_processor_d3d12.{cpp,h}`, `leia_display_processor_gl.{cpp,h}` | | Weaver (per API) | `leia_sr.{cpp,h}` (base + eye tracking), `leia_sr_d3d11.{cpp,h}`, `leia_sr_d3d12.{cpp,h}`, `leia_sr_gl.{cpp,h}` | | Background capture (transparency) | `leia_bg_capture_win.{cpp,h}` | +| 2D→3D lift (NeurD, D3D11) | `leia_lift_neurd.{cpp,h}`, `leia_neurd_abi.h` | | Shaders | `shaders/` | ## Docs in this directory @@ -21,6 +22,7 @@ Leia is the first 3D-display vendor integrated into DisplayXR. This directory do - **[Weaver internals](weaver.md)** — DX11 / DX12 / OpenGL / Vulkan weaver creation, inputs, weave() flow, DPI handling, phase math. - **[Transparency model](transparency.md)** — current primary path: WGC compose-under-bg on D3D11 / D3D12 / Vulkan. Replaces the older chroma-key approach for those APIs. - **[Chroma-key overlay (legacy / OpenGL fallback)](chroma-key-overlay.md)** — fallback path; still the only transparency path on the Leia OpenGL DP. +- **[2D→3D lift via NeurD](lift-neurd.md)** — the D3D11 DP's `lift_*` slots: dynamic NeurD.dll load, one-per-process activation (licence network failure = *activating*, retried), the shared-texture device bridge to NeurD's own D3D11 device, tracked-eye → NeurD viewpoint mapping, the `DXR_LEIA_LIFT*` knobs, and how to verify with `displayxr-cli lift caps|probe`. - **[Late latching](late-latching.md)** — which backends implement it (D3D11/GL automatic, VK needs our submit hook, D3D12 is a stub), what `srWeaverIsLateLatchingEnabled` hides, the once-per-frame / deferred-context contract from the shipped header, and the dot test — including why `pattern = 450` makes it lie. - **[SR panel readiness and late geometry](sr-readiness.md)** — the boot race (SR "default display" until the FPC serial link answers; never while the panel sleeps), the two readiness signals that actually work (`Global\sharedDeviceSerialMemory` count byte, a *fresh* valid display handle), the ONE shared startup budget (`DXR_LEIA_SR_READY_TIMEOUT_S`, default 20 s), the 1 Hz late-identification watcher that re-derives geometry and updates the live head device in place, and the `get_display_info` contract with the runtime (#266). - **[Window phase snapping](window-phase-snapping.md)** — the SR weaver's automatic `WndProc` subclassing for lenticular phase alignment during window drag (Windows), and the resolved `WndProcDispatcher` race. diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md new file mode 100644 index 0000000..b04fe36 --- /dev/null +++ b/docs/lift-neurd.md @@ -0,0 +1,226 @@ +# 2D→3D lift via NeurD (Windows, D3D11) + +The Leia D3D11 display processor implements the runtime's **lift** slots — convert a +single 2D RGBA frame into depth, a side-by-side stereo pair, or an N-view grid — by +driving Leia's **NeurD** library. Source: `src/drv_leia/leia_lift_neurd.{h,cpp}` +(module), `src/drv_leia/leia_neurd_abi.h` (NeurD binary interface), and the slot glue +at the bottom of `src/drv_leia/leia_display_processor_d3d11.cpp`. + +The runtime owns the lift *policy* — which content is lifted, the async worker thread, +the result mailbox/ring, and how the output reaches the compositor. The plug-in owns +only "turn this texture into that texture, now". + +## Contract (runtime side) + +Four slots appended to `struct xrt_display_processor_d3d11`, announced by +`XRT_DP_D3D11_HAS_LIFT` (runtime branch `feat/lift-ext`): + +| Slot | Plug-in behaviour | +|---|---| +| `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS\|NVIEW (1\|2\|4) once NeurD is present, else 0. `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 16, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | +| `lift_stream_create` | Non-blocking. Succeeds while NeurD is still activating — the NeurD stream is created lazily on the first convert. | +| `lift_stream_destroy` | Releases the stream's NeurD stream and bridge resources. | +| `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device, `DXGI_FORMAT_R8G8B8A8_UNORM`, owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | + +Every versioned struct is read only as far as its `struct_size` covers. + +Output layouts: DEPTH = one tile at the inference resolution; SBS = 2×1; NVIEW with +N views = `cols = N` for N ≤ 3, else `cols = ceil(sqrt(N))`, `rows = ceil(N/cols)`, +row-major (4 → 2×2). Extra grid slots repeat the last view. Tile size = the +*inference* resolution (after autoscaling), not the input size — use `out_w/out_h`. + +## Process model + +- **Dynamic load only.** `NeurD.dll` is found in NeurD's own loader order: `PATH` → + `NEURD_PATH` (full path to the DLL) → `HKLM\SOFTWARE\LeiaInc\NeurD` (default value = + install dir), loaded with `LOAD_WITH_ALTERED_SEARCH_PATH`. No NeurD header, import + lib or DLL is needed to build; the binary interface is mirrored in + `leia_neurd_abi.h` (append-only function table, every entry version-gated before it + is read). +- **Absent NeurD → nothing changes.** A presence probe (no `LoadLibrary`) runs on the + first caps/stream call; if the DLL is not found the process state becomes + *absent*, caps report `modes=0 / state=0`, and the plug-in behaves exactly as before. +- **One NeurD instance per process** (NeurD's rule). DP handles ref-count it. It is + loaded and initialised on a **detached background thread**, never on a caller's + thread and never at DP create (the DP factory runs on the service critical path). +- **Never de-initialised.** DPs are recreated on focus changes; re-init would re-run + licensing and model load. When the last DP handle goes the plug-in only calls + `NeurD_shrink_memory_pool`. + +### Licensing / activation + +`NeurD_init_with_options(multithreaded = TRUE)` runs NeurD's LexActivator licence +check. First activation on a machine needs the network (and WMI). Outcomes: + +| NeurD result | Plug-in state | Caps `state` | +|---|---|---| +| `SUCCESS` | ready | 2 | +| `LICENSE_NETWORK_ERROR` / `NETWORK_ERROR` | retry-wait; re-attempted on the next caps/stream/convert call ≥ 15 s later | 1 (activating) | +| `INVALID_LICENSE`, model/filesystem errors, anything else | failed for the process | 0 | + +Each transition logs one WARN (`Leia lift: ...`). + +### Backend + +`NeurD_set_backend` is called immediately before init; NeurD keeps a forced backend for +the process, so the **first** DP to activate NeurD decides it. The D3D11 path needs +NeurD's D3D11 device (`NeurD_get_dx_device`), which only the **DirectML** backend +provides — CUDA and OpenVINO return none, and lift then reports unavailable with a WARN +naming the fix. Hence the default is `directml`, not NeurD's own `auto`. + +## Threading + +- Every entry point may be called from any thread. `lift_convert` blocks for tens of + ms: the runtime calls it only from its lift thread. +- NeurD properties (output type, tile grid, gain, convergence, inpaint, autoscale) are + **process-global** inside NeurD, so one process-wide mutex makes *set properties → + convert → stage output* atomic across streams. Properties are re-sent only when they + change. +- The caller's immediate context must be `ID3D11Multithread`-protected (NeurD's and our + own copies run on it from the lift thread while the compositor renders). The plug-in + enables protection if it is off — the same mechanism the async weaver already relies + on (`enable_context_multithread_protection`, `leia_sr_d3d11.cpp`). NeurD's own + immediate context is protected too. + +## Device bridge + +NeurD runs on **its own** D3D11 device, possibly on a different adapter than the +runtime's service device on hybrid boxes. Its DX input must be a flat RGBA8 +`ID3D11Buffer` (stride `w*4`) with `D3D11_RESOURCE_MISC_SHARED`, and it hands the +output back on the input buffer's device. Buffers do not share reliably across devices; +textures do. So (the LeiaMeet `DxStereoConverter` pattern): + +``` +caller device NeurD device +------------- ------------ +input tex ──CopySubresourceRegion──▶ in_bridge (shared tex, created on NeurD's device) + event-query drain (CPU) + in_bridge SRV ──pack CS──▶ raw RGBA8 buffer ; drain + NeurD_convert_stream_dx[_interactive] (blocking) + NeurD output ──unpack CS / copy──▶ out_bridge ; drain +out_bridge (opened) ◀───────────────── returned to the runtime +``` + +Each hop is CPU-drained with an event query because NeurD's DirectML work runs on its +own queue, unordered against any D3D11 context. Bridges are per stream, sized lazily, +and rebuilt on a size/format/device change. The adapter LUIDs of both devices are +logged once per stream (`same-GPU copy` vs `cross-adapter copy`); the bridge is used +either way. Accepted inputs: single-sampled `R8G8B8A8_*` or `B8G8R8A8_*` textures +(the pack shader reads through a typed view, so BGRA is swizzled for free and sRGB +bytes pass through encoded — what NeurD expects). + +## Tracked eyes → NeurD viewpoints + +NeurD's `*_interactive` converts take one `(x, y, z)` triplet per output view in +**dimensionless model units**: `x` horizontal disparity gain, `y` vertical offset, `z` +depth offset. Its default stereo pattern is `x = −0.5 / +0.5`, i.e. one unit of `x` is +one nominal eye baseline. The plug-in maps display-space positions in **metres** (the +same eyes `get_predicted_eye_positions` reports) with: + +``` +x_n = clamp(G · x_m / 0.063, ±3) +y_n = clamp(G · y_m / 0.063, ±3) +z_n = 0 +G = DXR_LEIA_LIFT_VIEW_GAIN (default 1.0) +``` + +A centred viewer at 63 mm IPD therefore reproduces NeurD's default pattern exactly +(G = 1), and head motion becomes look-around parallax. `G = 0` pins every view to the +centre; `G > 1` exaggerates. Head `z` is deliberately not mapped: NeurD does not specify +how its `z` relates to viewing distance, so mapping it would be a guess. + +Viewpoint source, in order: + +1. **Explicit viewpoints** from the runtime (`viewpoints_xyz`, metres, display space): + `view_count` triplets are used 1:1; exactly two are treated as an eye pair. +2. **Tracked eyes** from the SR eye path, when tracking is live (inter-eye distance + > 1 mm — the same tracking-loss test as the eye slot). +3. **Untracked**: SBS uses NeurD's own default pattern (non-interactive convert); NVIEW + uses `x = i − (N−1)/2`. + +For N views from an eye pair, views are spaced one eye baseline apart and centred on +the eye midpoint. If NeurD predates the interactive API (< 0.4.5), SBS falls back to the +default pattern and NVIEW is refused. + +## Parameters + +| `xrt_dp_lift_params` | NeurD | +|---|---| +| `convergence < 0` | `AUTO_CONVERGENCE = TRUE` | +| `convergence ≥ 0` | `AUTO_CONVERGENCE = FALSE`, `CONVERGENCE` clamped to NeurD's [−0.2, 0.2] | +| `strength` | `GAIN_MULTIPLIER`, clamped [0.1, 10]; ≤ 0 → 1.0 | +| `inpaint` | 0 → `V1_STRETCH`, 1 → `V1_BLUR` (the DX video path supports V1 only) | +| `view_count` | NVIEW grid (2..16) | + +`xrt_dp_lift_stream_info.input_scale` in (0, 1] picks the smallest NeurD autoscaling +bucket (720p / 1080p / 1440p / none) that covers `input_scale × input height`; +otherwise the `DXR_LEIA_LIFT_SCALE` default applies. `content_hint` is recorded but +advisory: NeurD's DX stream path always runs its video model (photo mode is an +init-time, process-wide property). + +Depth output is NeurD's **relative** disparity, min-max normalised per frame at the +inference resolution. + +## Knobs + +Read once per DP at create (`leia_lift_neurd_create`). + +| Env | Default | Effect | +|---|---|---| +| `DXR_LEIA_LIFT` | on | `0` / `off` → caps `modes=0, state=0`; NeurD is never probed or loaded. | +| `DXR_LEIA_LIFT_BACKEND` | `directml` | `auto` \| `directml` \| `cuda` \| `openvino`. First activation in the process wins (NeurD's forced backend is sticky). Only DirectML yields a D3D11 device. | +| `DXR_LEIA_LIFT_SCALE` | `720` | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. NeurD's own default is 1440p; 720p is chosen for latency. Per-stream `input_scale` overrides. | +| `DXR_LEIA_LIFT_VIEW_GAIN` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | + +Under the service, remember these are read by `displayxr-service.exe`'s environment, +not the client's. + +## Building against the runtime + +The slots compile only when the runtime headers define `XRT_DP_D3D11_HAS_LIFT`. The +Windows pin `DXR_RUNTIME_GIT_TAG` is currently `v2.16.9`, which predates it — a pinned +build ships the plug-in with the lift slots compiled out (it logs +`lift slots NOT COMPILED`), and the runtime sees them absent via `struct_size`. + +To build with lift before the runtime tags it, use a local runtime checkout — nothing +is committed: + +```bat +set DXR_RUNTIME_SOURCE_DIR=C:\dev\displayxr-runtime.wt-lift +scripts\build-windows.bat build +``` + +or `cmake -DDXR_RUNTIME_GIT_TAG=feat/lift-ext ...` for a fetched build. **Do not commit +a branch or SHA pin**: `installer/CMakeLists.txt` derives `MIN_RUNTIME_VERSION` from +`DXR_RUNTIME_GIT_TAG`, and the orchestrator/meta-installer assume a `vX.Y.Z` tag. Once +the runtime ships the lift contract in a tag, move `DXR_RUNTIME_GIT_TAG` (and the +workflow's runtime ref, which the rule-5 check keeps equal) to that tag in the plug-in +release that should carry lift. Because the slots are appended (ADR-020), that re-pin +raises the installer floor to the lift tag — the price of shipping the capability. + +## Verifying + +On a Windows box with NeurD installed (`HKLM\SOFTWARE\LeiaInc\NeurD` present) and the +lift-enabled runtime + this plug-in registered: + +1. `displayxr-cli lift caps` — first call reports `state=activating` (the background + worker is loading NeurD); repeat until `state=ready`, `modes=0x7`, + `backend=neurd-directml`. The runtime log shows `Leia lift: loaded NeurD x.y.z from + ...` and `Leia lift: NeurD READY`. +2. `displayxr-cli lift probe` — runs a stream create + convert + destroy. Check the log + for the adapter-LUID line and no `Leia lift:` WARN after it; the probe's reported + latency should sit near 22 ms (DirectML) at 720p. +3. Absent-NeurD check: `set DXR_LEIA_LIFT=0` (or rename the NeurD install key) → + `lift caps` reports `modes=0 state=unavailable`, and a normal + `cube_handle_d3d11_win` session weaves exactly as before. +4. Licence path: first activation offline → `state=activating` with a + `LICENSE_NETWORK_ERROR` WARN; reconnect → ready within ~15 s of the next call. + +## Known limits + +- Windows / D3D11 only. The D3D12, GL and Vulkan DPs do not implement lift. +- The activation thread is detached; unloading the plug-in DLL while it runs + (a first activation in progress) is unsafe. The runtime does not unload plug-ins + mid-session today. +- NeurD's 32-stream limit is process-wide and shared with anything else in the + service process that uses NeurD. From 5a9615895b5d186ae29cb84c20da30df3e66c1c3 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 13:20:05 -0700 Subject: [PATCH 04/21] fix(lift): caps never waits behind an in-flight convert Co-Authored-By: Claude Opus 5.5 (1M context) --- src/drv_leia/leia_lift_neurd.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index cae58e0..b51ca26 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -1145,7 +1145,9 @@ leia_lift_neurd_get_caps(struct leia_lift_neurd *l, struct leia_lift_neurd_caps out->depth_semantics = 0; // relative: per-frame min-max normalised disparity uint64_t lat = g.latency_ns.load(); if (s == G_READY) { - std::lock_guard lock(g.mtx); + // No lock: backend/backend_name are written once, before the seq_cst + // store that publishes G_READY, and never again — caps must not wait + // behind a convert holding g.mtx. snprintf(out->backend, sizeof(out->backend), "%s", g.backend_name); if (lat == 0) { lat = (g.backend == LEIA_NEURD_BACKEND_DIRECTML) ? kPriorLatencyDirectMlNs From 8ce86fe617392b810911d55f85be2dc680a68079 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 13:27:47 -0700 Subject: [PATCH 05/21] fix(lift): match the runtime's landed lift contract (xrt_dp_lift.h) - NVIEW is one row, view 0 leftmost (was a square grid); max_views 8 so the row fits D3D11's 16384 limit at 1440p. - DEPTH returns a single-channel R8_UNORM texture (R8 unpack shader). - strength 0 = flat (only negative falls back to 1.0); inpaint non-zero = blur fill. - input_scale 1 = native; DXR_LEIA_LIFT_SCALE, when set, overrides streams. - lift_convert_blob (GAUSSIANS) left NULL. - docs: contract deltas, convergence-units gap, lift-DP weaver caveat. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/lift-neurd.md | 56 +++++++++++++------ src/drv_leia/leia_lift_neurd.cpp | 93 +++++++++++++++++++++----------- src/drv_leia/leia_lift_neurd.h | 10 ++-- 3 files changed, 108 insertions(+), 51 deletions(-) diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index b04fe36..762ac5a 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -12,22 +12,27 @@ only "turn this texture into that texture, now". ## Contract (runtime side) -Four slots appended to `struct xrt_display_processor_d3d11`, announced by -`XRT_DP_D3D11_HAS_LIFT` (runtime branch `feat/lift-ext`): +Slots appended to `struct xrt_display_processor_d3d11` (runtime `xrt_dp_lift.h`, +ADR-042), announced by `XRT_DP_D3D11_HAS_LIFT` (runtime branch `feat/lift-ext`). The +plug-in fills the four texture-mode slots; the fifth, `lift_convert_blob` +(GAUSSIANS, photo → splats), is left NULL — NeurD has no splat path — so `modes` never +carries bit 8. | Slot | Plug-in behaviour | |---|---| -| `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS\|NVIEW (1\|2\|4) once NeurD is present, else 0. `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 16, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | +| `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS\|NVIEW (1\|2\|4) once NeurD is present, else 0. `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 8, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | | `lift_stream_create` | Non-blocking. Succeeds while NeurD is still activating — the NeurD stream is created lazily on the first convert. | | `lift_stream_destroy` | Releases the stream's NeurD stream and bridge resources. | -| `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device, `DXGI_FORMAT_R8G8B8A8_UNORM`, owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | +| `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device — `R8G8B8A8_UNORM` for SBS/NVIEW, `R8_UNORM` for DEPTH — owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | Every versioned struct is read only as far as its `struct_size` covers. -Output layouts: DEPTH = one tile at the inference resolution; SBS = 2×1; NVIEW with -N views = `cols = N` for N ≤ 3, else `cols = ceil(sqrt(N))`, `rows = ceil(N/cols)`, -row-major (4 → 2×2). Extra grid slots repeat the last view. Tile size = the -*inference* resolution (after autoscaling), not the input size — use `out_w/out_h`. +Output layouts (runtime contract): DEPTH = one channel at the inference resolution +(NeurD writes the normalised disparity replicated into RGB; the unpack keeps R); +SBS = 2 views side by side; NVIEW = `view_count` views side by side in **one row**, +view 0 leftmost. Tile size = the *inference* resolution (after autoscaling), not the +input size — use `out_w/out_h`. One row is why `max_views` is 8: 8 × 2560 (1440p) is +the widest that fits D3D11's 16384-texel limit; a wider request fails with a WARN. ## Process model @@ -131,7 +136,7 @@ how its `z` relates to viewing distance, so mapping it would be a guess. Viewpoint source, in order: -1. **Explicit viewpoints** from the runtime (`viewpoints_xyz`, metres, display space): +1. **Explicit viewpoints** from the runtime (recommended — see *Known limits*) (`viewpoints_xyz`, metres, display space): `view_count` triplets are used 1:1; exactly two are treated as an eye pair. 2. **Tracked eyes** from the SR eye path, when tracking is live (inter-eye distance > 1 mm — the same tracking-loss test as the eye slot). @@ -148,13 +153,19 @@ default pattern and NVIEW is refused. |---|---| | `convergence < 0` | `AUTO_CONVERGENCE = TRUE` | | `convergence ≥ 0` | `AUTO_CONVERGENCE = FALSE`, `CONVERGENCE` clamped to NeurD's [−0.2, 0.2] | -| `strength` | `GAIN_MULTIPLIER`, clamped [0.1, 10]; ≤ 0 → 1.0 | -| `inpaint` | 0 → `V1_STRETCH`, 1 → `V1_BLUR` (the DX video path supports V1 only) | -| `view_count` | NVIEW grid (2..16) | - -`xrt_dp_lift_stream_info.input_scale` in (0, 1] picks the smallest NeurD autoscaling -bucket (720p / 1080p / 1440p / none) that covers `input_scale × input height`; -otherwise the `DXR_LEIA_LIFT_SCALE` default applies. `content_hint` is recorded but +| `strength` | `GAIN_MULTIPLIER` (1 = NeurD's calibrated budget, 0 = flat), clamped [0, 10]; negative → 1.0 | +| `inpaint` | NeurD always fills disocclusions: 0 → `V1_STRETCH` (cheapest), non-zero → `V1_BLUR` (the DX video path supports V1 only) | +| `view_count` | NVIEW row width (2..8) | + +**Convergence units are not reconciled yet.** The runtime defines `convergence` as +"relative depth placed at the display plane"; NeurD's `CONVERGENCE` is its own +disparity offset in [−0.2, 0.2]. The value is passed through clamped; the mapping needs +calibrating on a panel (auto-convergence, the default, is unaffected). + +`xrt_dp_lift_stream_info.input_scale` in (0, 1] (1 = native) picks the smallest NeurD +autoscaling bucket (720p / 1080p / 1440p / none) that covers `input_scale × input +height`; outside that range the 720p default applies, and an explicitly set +`DXR_LEIA_LIFT_SCALE` overrides every stream. `content_hint` is recorded but advisory: NeurD's DX stream path always runs its video model (photo mode is an init-time, process-wide property). @@ -169,7 +180,7 @@ Read once per DP at create (`leia_lift_neurd_create`). |---|---|---| | `DXR_LEIA_LIFT` | on | `0` / `off` → caps `modes=0, state=0`; NeurD is never probed or loaded. | | `DXR_LEIA_LIFT_BACKEND` | `directml` | `auto` \| `directml` \| `cuda` \| `openvino`. First activation in the process wins (NeurD's forced backend is sticky). Only DirectML yields a D3D11 device. | -| `DXR_LEIA_LIFT_SCALE` | `720` | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. NeurD's own default is 1440p; 720p is chosen for latency. Per-stream `input_scale` overrides. | +| `DXR_LEIA_LIFT_SCALE` | unset (→ stream `input_scale`, else 720p) | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. When set it overrides every stream's `input_scale`. NeurD's own default is 1440p; 720p is the fallback for latency. | | `DXR_LEIA_LIFT_VIEW_GAIN` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | Under the service, remember these are read by `displayxr-service.exe`'s environment, @@ -218,6 +229,17 @@ lift-enabled runtime + this plug-in registered: ## Known limits +- **The runtime's dedicated lift DP gets a full SR weaver.** The runtime calls the + lift slots on a DP it creates *for* lift, with a NULL window, and asks that such a DP + be cheap. This plug-in cannot tell that DP from a NULL-HWND `_texture` session DP + (which must weave), so it creates the usual async SR weaver for it. Two + consequences: (1) a second weaver on the shared SR context may briefly perturb the + lens (the effect the #625 snap probe guards against), and (2) the tracked eyes the + lift reads come from that extra weaver. Recommended: the runtime passes explicit + `viewpoints_xyz` (the panel DP's predicted eyes) on every convert, and a follow-up + gives the DP a lift-only signal (or makes NULL-window weaver creation lazy, on the + first `process_atlas`) so the lift DP never builds a weaver. + - Windows / D3D11 only. The D3D12, GL and Vulkan DPs do not implement lift. - The activation thread is detached; unloading the plug-in DLL while it runs (a first activation in progress) is unsafe. The runtime does not unload plug-ins diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index b51ca26..290f463 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -86,7 +86,10 @@ constexpr float kIpdRefM = 0.063f; constexpr float kViewpointClamp = 3.0f; //! NeurD's MAX_STREAMS. constexpr uint32_t kMaxStreams = 32; -constexpr uint32_t kMaxViews = 16; +//! N views go side by side in ONE row (runtime contract), so the output is +//! N x inference-width wide; 8 x 2560 (1440p) still fits D3D11's 16384 limit. +constexpr uint32_t kMaxViews = 8; +constexpr uint32_t kMaxTexDim = D3D11_REQ_TEXTURE2D_U_OR_V_DIMENSION; //! Retry period for a licence activation that failed on the network. constexpr uint64_t kActivationRetryMs = 15000; //! Upper bound for any CPU drain of a GPU queue (device-lost guard). @@ -170,6 +173,7 @@ struct knobs bool enabled; //!< DXR_LEIA_LIFT (default on) int backend; //!< DXR_LEIA_LIFT_BACKEND (default directml) int32_t autoscaling; //!< DXR_LEIA_LIFT_SCALE (default 720p) + bool scale_forced; //!< DXR_LEIA_LIFT_SCALE was set: overrides the stream's input_scale float view_gain; //!< DXR_LEIA_LIFT_VIEW_GAIN (default 1.0) }; @@ -212,6 +216,7 @@ read_knobs() e = std::getenv("DXR_LEIA_LIFT_SCALE"); if (e != nullptr && e[0] != '\0') { + k.scale_forced = true; if (env_ieq(e, "none") || env_ieq(e, "native") || env_ieq(e, "0")) { k.autoscaling = LEIA_NEURD_AUTOSCALING_NONE; } else if (env_ieq(e, "720") || env_ieq(e, "720p")) { @@ -272,6 +277,20 @@ void main(uint3 id : SV_DispatchThreadID) } )"; +// NeurD's RGBA8 depth buffer -> single-channel R8 bridge (DEPTH contract: one +// channel). NeurD writes the normalised disparity replicated into RGB; R is kept. +const char *kUnpackR8Cs = R"( +cbuffer P : register(b0) { uint W; uint H; uint StrideBytes; uint Pad; }; +RWByteAddressBuffer Src : register(u0); +RWTexture2D Dst : register(u1); +[numthreads(16, 16, 1)] +void main(uint3 id : SV_DispatchThreadID) +{ + if (id.x >= W || id.y >= H) return; + Dst[id.xy] = (float)(Src.Load(id.y * StrideBytes + id.x * 4) & 0xFF) / 255.0; +} +)"; + /* * * Process-wide NeurD instance. @@ -311,6 +330,7 @@ struct global ID3D11Query *nd_done = nullptr; ID3D11ComputeShader *cs_pack = nullptr; ID3D11ComputeShader *cs_unpack = nullptr; + ID3D11ComputeShader *cs_unpack_r8 = nullptr; ID3D11Buffer *cb = nullptr; LUID nd_luid = {}; @@ -496,16 +516,18 @@ setup_nd_device_locked() D3D11_QUERY_DESC qd = {D3D11_QUERY_EVENT, 0}; g.cs_pack = compile_cs(g.nd_dev, kPackCs, "leia_lift_pack"); g.cs_unpack = compile_cs(g.nd_dev, kUnpackCs, "leia_lift_unpack"); + g.cs_unpack_r8 = compile_cs(g.nd_dev, kUnpackR8Cs, "leia_lift_unpack_r8"); D3D11_BUFFER_DESC bd = {}; bd.ByteWidth = 16; bd.Usage = D3D11_USAGE_DYNAMIC; bd.BindFlags = D3D11_BIND_CONSTANT_BUFFER; bd.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; - if (g.cs_pack == nullptr || g.cs_unpack == nullptr || FAILED(g.nd_dev->CreateQuery(&qd, &g.nd_done)) || + if (g.cs_pack == nullptr || g.cs_unpack == nullptr || g.cs_unpack_r8 == nullptr || FAILED(g.nd_dev->CreateQuery(&qd, &g.nd_done)) || FAILED(g.nd_dev->CreateBuffer(&bd, nullptr, &g.cb))) { U_LOG_W("Leia lift: bridge setup on NeurD's device failed — lift unavailable"); safe_release(g.cs_pack); safe_release(g.cs_unpack); + safe_release(g.cs_unpack_r8); safe_release(g.nd_done); safe_release(g.cb); safe_release(g.nd_ctx); @@ -728,6 +750,7 @@ struct lift_stream // Output bridge. uint32_t out_w, out_h; + DXGI_FORMAT out_fmt; //!< R8G8B8A8_UNORM, or R8_UNORM for DEPTH ID3D11Texture2D *out_nd; ID3D11Texture2D *out_ours; ID3D11UnorderedAccessView *out_uav; @@ -755,6 +778,7 @@ release_out_bridge(lift_stream *s) safe_release(s->out_ours); safe_release(s->out_nd); s->out_w = s->out_h = 0; + s->out_fmt = DXGI_FORMAT_UNKNOWN; } //! Called with g.mtx held. @@ -848,13 +872,19 @@ ensure_in_bridge(lift_stream *s, uint32_t w, uint32_t h, DXGI_FORMAT family) } bool -ensure_out_bridge(lift_stream *s, uint32_t w, uint32_t h) +ensure_out_bridge(lift_stream *s, uint32_t w, uint32_t h, DXGI_FORMAT fmt) { - if (s->out_nd != nullptr && s->out_w == w && s->out_h == h) { + if (s->out_nd != nullptr && s->out_w == w && s->out_h == h && s->out_fmt == fmt) { return true; } release_out_bridge(s); - if (!make_shared_tex(s->dev, w, h, DXGI_FORMAT_R8G8B8A8_UNORM, + if (w > kMaxTexDim || h > kMaxTexDim) { + LIFT_WARN_ONCE("Leia lift: output %ux%u exceeds the D3D11 texture limit (%u) — lower view_count or " + "DXR_LEIA_LIFT_SCALE", + w, h, kMaxTexDim); + return false; + } + if (!make_shared_tex(s->dev, w, h, fmt, D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_UNORDERED_ACCESS, &s->out_nd, &s->out_ours) || FAILED(g.nd_dev->CreateUnorderedAccessView(s->out_nd, nullptr, &s->out_uav))) { LIFT_WARN_ONCE("Leia lift: output bridge texture %ux%u failed", w, h); @@ -863,6 +893,7 @@ ensure_out_bridge(lift_stream *s, uint32_t w, uint32_t h) } s->out_w = w; s->out_h = h; + s->out_fmt = fmt; return true; } @@ -931,12 +962,15 @@ stage_output(lift_stream *s, const struct leia_neurd_image &out) } const uint32_t w = (uint32_t)out.width; const uint32_t h = (uint32_t)out.height; - if (!ensure_out_bridge(s, w, h)) { - return false; - } ID3D11Texture2D *tex = nullptr; if (SUCCEEDED(res->QueryInterface(__uuidof(ID3D11Texture2D), (void **)&tex)) && tex != nullptr) { + // A texture result is copied as-is (RGBA8 even for DEPTH; out_format + // says so). NeurD's stream path returns buffers; this is defensive. + if (!ensure_out_bridge(s, w, h, DXGI_FORMAT_R8G8B8A8_UNORM)) { + tex->Release(); + return false; + } // Region copy: NeurD's internal texture may exceed the reported size. D3D11_BOX box = {0, 0, 0, w, h, 1}; g.nd_ctx->CopySubresourceRegion(s->out_nd, 0, 0, 0, 0, tex, 0, &box); @@ -949,6 +983,11 @@ stage_output(lift_stream *s, const struct leia_neurd_image &out) LIFT_WARN_ONCE("Leia lift: NeurD output is neither a texture nor a buffer"); return false; } + const bool depth = (s->mode == LEIA_LIFT_MODE_DEPTH); + if (!ensure_out_bridge(s, w, h, depth ? DXGI_FORMAT_R8_UNORM : DXGI_FORMAT_R8G8B8A8_UNORM)) { + buf->Release(); + return false; + } D3D11_BUFFER_DESC bd = {}; buf->GetDesc(&bd); uint32_t stride = (out.stride >= (int32_t)(w * 4)) ? (uint32_t)out.stride : w * 4; @@ -978,7 +1017,7 @@ stage_output(lift_stream *s, const struct leia_neurd_image &out) return false; } ID3D11UnorderedAccessView *uavs[2] = {s->out_src_uav, s->out_uav}; - g.nd_ctx->CSSetShader(g.cs_unpack, nullptr, 0); + g.nd_ctx->CSSetShader(depth ? g.cs_unpack_r8 : g.cs_unpack, nullptr, 0); g.nd_ctx->CSSetConstantBuffers(0, 1, &g.cb); g.nd_ctx->CSSetUnorderedAccessViews(0, 2, uavs, nullptr); g.nd_ctx->Dispatch(groups16(w), groups16(h), 1); @@ -1019,25 +1058,14 @@ prop_f(enum leia_neurd_prop p, float v, float &cache) return true; } -//! Grid for N views: cols = ceil(sqrt(N)), rows = ceil(N/cols), row-major. -void -nview_grid(uint32_t n, uint32_t *cols, uint32_t *rows) -{ - if (n <= 3) { - *cols = n; - *rows = 1; - return; - } - uint32_t c = (uint32_t)std::ceil(std::sqrt((double)n)); - *cols = c; - *rows = (n + c - 1) / c; -} int32_t -autoscale_for(const lift_stream *s, int32_t knob_default, uint32_t in_h) +autoscale_for(const lift_stream *s, const struct knobs &k, uint32_t in_h) { - if (!(s->input_scale > 0.0f) || s->input_scale > 1.0f) { - return knob_default; + // An explicit DXR_LEIA_LIFT_SCALE is a developer override and wins; else + // the stream's input_scale (1 = native) picks the bucket; else the default. + if (k.scale_forced || !(s->input_scale > 0.0f) || s->input_scale > 1.0f) { + return k.autoscaling; } // Smallest NeurD bucket that still covers the requested inference height. float target = s->input_scale * (float)in_h; @@ -1381,18 +1409,23 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, views = 1; cols = rows = 1; } else if (s->mode == LEIA_LIFT_MODE_NVIEW) { + // Runtime contract: N views side by side in ONE row, view 0 leftmost. views = std::min(std::max(dp.view_count, 2u), kMaxViews); - nview_grid(views, &cols, &rows); + cols = views; + rows = 1; } const bool auto_conv = !(dp.convergence >= 0.0f); - const float gain = (dp.strength > 0.0f) ? clampf(dp.strength, 0.1f, 10.0f) : 1.0f; - const int32_t inpaint = (dp.inpaint == 1) ? LEIA_NEURD_INPAINT_V1_BLUR : LEIA_NEURD_INPAINT_V1_STRETCH; + // strength: 1 = NeurD's calibrated budget, 0 = flat; negative/NaN = default. + const float gain = (dp.strength >= 0.0f) ? clampf(dp.strength, 0.0f, 10.0f) : 1.0f; + // NeurD always fills disocclusions; non-zero picks the blur fill, 0 the + // cheaper edge stretch. + const int32_t inpaint = (dp.inpaint != 0) ? LEIA_NEURD_INPAINT_V1_BLUR : LEIA_NEURD_INPAINT_V1_STRETCH; bool props_ok = prop_i(LEIA_NEURD_PROP_OUTPUT_TYPE, out_type, g.p_out_type) && prop_i(LEIA_NEURD_PROP_OUTPUT_TILES_W, (int32_t)cols, g.p_tiles_w) && prop_i(LEIA_NEURD_PROP_OUTPUT_TILES_H, (int32_t)rows, g.p_tiles_h) && prop_i(LEIA_NEURD_PROP_INPAINT_TYPE, inpaint, g.p_inpaint) && - prop_i(LEIA_NEURD_PROP_INPUT_AUTOSCALING, autoscale_for(s, l->k.autoscaling, h), + prop_i(LEIA_NEURD_PROP_INPUT_AUTOSCALING, autoscale_for(s, l->k, h), g.p_autoscale) && prop_i(LEIA_NEURD_PROP_AUTO_CONVERGENCE, auto_conv ? 1 : 0, g.p_autoconv) && prop_f(LEIA_NEURD_PROP_GAIN_MULTIPLIER, gain, g.p_gain); @@ -1497,7 +1530,7 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, *out_resource = static_cast(s->out_ours); *out_w = s->out_w; *out_h = s->out_h; - *out_format = (uint32_t)DXGI_FORMAT_R8G8B8A8_UNORM; + *out_format = (uint32_t)s->out_fmt; } catch (...) { // std::vector / std::mutex can throw; nothing may escape into the C runtime. LIFT_WARN_ONCE("Leia lift: exception in convert — frame dropped"); diff --git a/src/drv_leia/leia_lift_neurd.h b/src/drv_leia/leia_lift_neurd.h index 19dc65c..1b13a29 100644 --- a/src/drv_leia/leia_lift_neurd.h +++ b/src/drv_leia/leia_lift_neurd.h @@ -60,14 +60,15 @@ struct leia_lift_neurd_stream_desc { uint32_t mode; //!< One LEIA_LIFT_MODE_* value. uint32_t content_hint; //!< 0 video, 1 photo (advisory; the DX path is video-model only). - float input_scale; //!< (0,1] fraction of input height to infer at; <=0 = knob default. + float input_scale; //!< (0,1] fraction of input height to infer at (1 = native); + //!< outside that, or DXR_LEIA_LIFT_SCALE set = the knob. }; struct leia_lift_neurd_params { float convergence; //!< <0 = NeurD auto-convergence; else NeurD units, clamped to [-0.2, 0.2]. - float strength; //!< NeurD gain multiplier; <=0 = 1.0; clamped to [0.1, 10]. - uint32_t inpaint; //!< 0 stretch, 1 blur. + float strength; //!< NeurD gain multiplier: 1 = calibrated budget, 0 = flat; <0 = 1.0; max 10. + uint32_t inpaint; //!< 0 stretch fill, non-zero blur fill. uint32_t view_count; //!< NVIEW only (2..max_views). }; @@ -114,7 +115,8 @@ leia_lift_neurd_stream_destroy(struct leia_lift_neurd *lift, uint64_t id); * default pattern is used. * @param out_resource ID3D11Texture2D* on the caller's device, owned by the * stream, valid until the next convert on this stream. - * @param out_format DXGI_FORMAT (always R8G8B8A8_UNORM today). + * @param out_format DXGI_FORMAT: R8G8B8A8_UNORM (SBS / NVIEW, N views in one + * row, view 0 leftmost) or R8_UNORM (DEPTH). */ bool leia_lift_neurd_convert(struct leia_lift_neurd *lift, From 8d39657c92d0bf96c6cb3f23ac2a8eb1e41e668b Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 13:27:57 -0700 Subject: [PATCH 06/21] chore(lift): assert lift mode/state values against the runtime's Co-Authored-By: Claude Opus 5.5 (1M context) --- src/drv_leia/leia_display_processor_d3d11.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/drv_leia/leia_display_processor_d3d11.cpp b/src/drv_leia/leia_display_processor_d3d11.cpp index 5cddc70..fb1a319 100644 --- a/src/drv_leia/leia_display_processor_d3d11.cpp +++ b/src/drv_leia/leia_display_processor_d3d11.cpp @@ -2314,6 +2314,14 @@ leia_dp_d3d11_set_window(struct xrt_display_processor_d3d11 *xdp, void *window_h * struct_size covers (ADR-020 append-only rule), so a runtime that grows these * structs keeps working against this plug-in and vice versa. */ +static_assert(LEIA_LIFT_MODE_DEPTH == XRT_DP_LIFT_MODE_DEPTH && LEIA_LIFT_MODE_SBS == XRT_DP_LIFT_MODE_SBS && + LEIA_LIFT_MODE_NVIEW == XRT_DP_LIFT_MODE_NVIEW, + "lift mode bits must match the runtime's XRT_DP_LIFT_MODE_*"); +static_assert(LEIA_LIFT_STATE_UNAVAILABLE == XRT_DP_LIFT_STATE_UNAVAILABLE && + LEIA_LIFT_STATE_ACTIVATING == XRT_DP_LIFT_STATE_ACTIVATING && + LEIA_LIFT_STATE_READY == XRT_DP_LIFT_STATE_READY, + "lift states must match the runtime's XRT_DP_LIFT_STATE_*"); + #define LEIA_LIFT_COVERS(ptr, type, field) \ ((ptr)->struct_size >= offsetof(type, field) + sizeof(((type *)0)->field)) From cb6537ad4c901eba1ba714e98455d4493e2a888b Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 13:34:18 -0700 Subject: [PATCH 07/21] refactor(lift): build against the real NeurD headers, fetched from private media_sdk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove src/drv_leia/leia_neurd_abi.h. It was a hand-written mirror of NeurD's public header — enum values, struct layouts and the function-table order only; no NeurD code was ever included. Leia's NeurD header is proprietary, so the public plug-in now fetches the REAL headers at build time instead, the same way it fetches the SR SDK: - NEURD_SDK_REF (v0.4.6, a media_sdk release tag; headers byte-identical to dev@25a713d93) + NEURD_SDK_REPO pinned in scripts/build-windows.bat and build-windows.yml, kept equal by check_sr_pins.py. - scripts/fetch-neurd-headers.ps1 pulls NeurD.h, _NeurD_detail.h, _NeurD_table.h via gh (CI: LEIALOFT_GITHUB_TOKEN) and generates NeurD_version.h from NeurD_version.h.in + the ref's project VERSION, into the gitignored NeurD-SDK-/include. - Fetch failure is SOFT everywhere: CMake compiles the lift module out (DXR_LEIA_HAS_NEURD undefined), the DP's lift slots stay NULL, the runtime reports lift unavailable, the rest of the plug-in is unchanged. - leia_lift_neurd.cpp still loads NeurD.dll dynamically (GetProcAddress NeurD_load, PFN_NeurD_load) and calls through the returned table using the real header's version-gated inline wrappers — no import lib, and any NeurD signature drift is now a compile error. - docs/lift-neurd.md + CLAUDE.md: private-repo access prerequisite, pins, soft-failure behaviour, win-box build steps. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/build-windows.yml | 23 ++ .gitignore | 2 + CLAUDE.md | 11 +- docs/README.md | 2 +- docs/lift-neurd.md | 48 +++- scripts/build-windows.bat | 31 +++ scripts/check_sr_pins.py | 5 +- scripts/fetch-neurd-headers.ps1 | 64 +++++ src/drv_leia/CMakeLists.txt | 36 ++- src/drv_leia/leia_display_processor_d3d11.cpp | 14 +- src/drv_leia/leia_lift_neurd.cpp | 237 +++++++++--------- src/drv_leia/leia_neurd_abi.h | 223 ---------------- 12 files changed, 342 insertions(+), 354 deletions(-) create mode 100644 scripts/fetch-neurd-headers.ps1 delete mode 100644 src/drv_leia/leia_neurd_abi.h diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index ca9f20d..9cb33c2 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -88,6 +88,12 @@ jobs: # Read with LEIALOFT_GITHUB_TOKEN — the same secret the Android arm uses # for LeiaInc/CNSDK. Fork PRs have no secret and cannot build this arm. SR_SDK_REPO: LeiaInc/SR-SDK-Windows-Releases-Internal-Public + # NeurD headers for the 2D->3D lift module (docs/lift-neurd.md), fetched + # from the PRIVATE LeiaInc/media_sdk repo into the gitignored + # NeurD-SDK-/include -- never committed. Pin a media_sdk release tag. + # KEEP IN SYNC with scripts/build-windows.bat (check_sr_pins.py asserts). + NEURD_SDK_REF: v0.4.6 + NEURD_SDK_REPO: LeiaInc/media_sdk GH_TOKEN: ${{ github.token }} # SourceForge serves a bot-check page to CI runners, so vcpkg's pthreads # port (vcpkg_from_sourceforge) fails its hash on every mirror. This @@ -180,6 +186,21 @@ jobs: if (-not (Test-Path "$env:SR_V2_DIR/$f")) { throw "SR v2 SDK missing $f" } } + # SOFT: without the headers the lift module compiles out and the plug-in + # is otherwise identical, so a missing token or an unreachable repo must + # not fail the build -- it is surfaced as a warning annotation instead. + - name: Fetch NeurD headers + if: needs.DetectChanges.outputs.docs_only != 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ secrets.LEIALOFT_GITHUB_TOKEN }} + run: | + ./scripts/fetch-neurd-headers.ps1 -Repo $env:NEURD_SDK_REPO -Ref $env:NEURD_SDK_REF -Dest "NeurD-SDK-$env:NEURD_SDK_REF" + if ($LASTEXITCODE -ne 0) { + Write-Host "::warning::NeurD headers not fetched ($env:NEURD_SDK_REPO@$env:NEURD_SDK_REF) -- this build ships WITHOUT 2D->3D lift." + exit 1 + } + # The plug-in CMake adds the runtime as a subdirectory, so we need # all the runtime's own build prereqs (vcpkg, Vulkan SDK, OpenXR # loader). Same shape as the runtime's build-windows.yml. @@ -282,6 +303,8 @@ jobs: env: LEIASR_SDKROOT: ${{ github.workspace }}/LeiaSR-SDK-${{ env.SR_TAG }}-win64 LEIASR_V2_SDKROOT: ${{ github.workspace }}/${{ env.SR_V2_DIR }} + # Read by src/drv_leia/CMakeLists.txt; absent dir => lift compiles out. + NEURD_SDK_ROOT: ${{ github.workspace }}/NeurD-SDK-${{ env.NEURD_SDK_REF }} run: | cmake -S . -B build -G "Ninja Multi-Config" ` -DDXR_RUNTIME_SOURCE_DIR="${{ github.workspace }}/displayxr-runtime" ` diff --git a/.gitignore b/.gitignore index e55eba9..a813734 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ __MACOSX/ # Fetched SR SDK (not committed; downloaded by CI / scripts/build-windows.bat) LeiaSR-SDK-*/ simulatedreality-SDK-*/ +# Fetched NeurD headers (Leia-private; scripts/fetch-neurd-headers.ps1) +NeurD-SDK-*/ # Installer artifacts (built into _package/) DisplayXRLeiaSRSetup-*.exe diff --git a/CLAUDE.md b/CLAUDE.md index 244c6db..09aa1fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,7 +98,7 @@ ADR-020 spec: [`displayxr-runtime/docs/adr/ADR-020-plugin-abi-policy.md`](https: | `src/drv_leia/leia_display_processor*.{cpp,h}` | Per-API display processors (D3D11, D3D12, GL). Implement `xrt_display_processor_` vtables; the runtime's compositor invokes them per-frame. | | `src/drv_leia/leia_sr_*.{cpp,h}` | SR SDK weaver wrappers. The SDK throws `std::runtime_error` as routine internal control flow (~11/frame in some paths) — every call wrapped in try/catch. | | `src/drv_leia/leia_bg_capture_win.{cpp,h}` | WGC background capture for compose-under transparency (Leia transparency model). | -| `src/drv_leia/leia_lift_neurd.{cpp,h}`, `leia_neurd_abi.h` | 2D→3D lift slots on the D3D11 DP, backed by a dynamically loaded NeurD.dll (absent ⇒ caps unavailable, nothing loaded). Slots compile only against runtime headers with `XRT_DP_D3D11_HAS_LIFT`. See [`docs/lift-neurd.md`](docs/lift-neurd.md). | +| `src/drv_leia/leia_lift_neurd.{cpp,h}` | 2D→3D lift slots on the D3D11 DP, backed by a dynamically loaded NeurD.dll (absent ⇒ caps unavailable, nothing loaded). Compiles only with the private NeurD headers (fetched at build time, below) AND runtime headers with `XRT_DP_D3D11_HAS_LIFT`; otherwise the slots stay NULL. See [`docs/lift-neurd.md`](docs/lift-neurd.md). | | `src/drv_leia/leia_edid_probe.c` | EDID-based hardware detection — answers "is a Leia display attached?" before the SR SDK initializes. | | `src/drv_leia_linux/leia_sr_linux.h` | **Linux weaver-backend seam** — interface shaped 1:1 by `docs/leia-linux-sdk-contract.md` (every declaration cites its R-* requirement). Track B implements it against the real SDK. | | `src/drv_leia_linux/leia_sr_stub.c` | Track A stub backend: canned panel info + passthrough SBS blit, `TODO(Track B)` at every body. | @@ -135,6 +135,15 @@ run by `.github/workflows/lint.yml` on every PR, asserts all five pins declared exactly once and identically in both files, and that `SR_V2_TAG`/`SR_V2_DIR` name the same SR v2 build. +**NeurD headers (2D→3D lift) follow the same rule.** They are Leia-private +and come from the private **`LeiaInc/media_sdk`** repo at `NEURD_SDK_REF` +(pinned next to the SR pins in both files, also checked by +`check_sr_pins.py`), fetched by `scripts/fetch-neurd-headers.ps1` into the +gitignored `NeurD-SDK-/`. **Never commit or mirror them.** Your `gh` +account (and CI's `LEIALOFT_GITHUB_TOKEN`) needs read access to +`media_sdk`; without it the fetch fails SOFT and the plug-in builds without +lift. Details: [`docs/lift-neurd.md`](docs/lift-neurd.md). + ### Linux ```bash ./scripts/build-linux.sh # Track A stub: .so + displayxr-cli, stage manifest, selftest diff --git a/docs/README.md b/docs/README.md index 213fb2d..8cc3d13 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,7 +14,7 @@ Leia is the first 3D-display vendor integrated into DisplayXR. This directory do | Display processor (per API) | `leia_display_processor.{cpp,h}` (base), `leia_display_processor_d3d11.{cpp,h}`, `leia_display_processor_d3d12.{cpp,h}`, `leia_display_processor_gl.{cpp,h}` | | Weaver (per API) | `leia_sr.{cpp,h}` (base + eye tracking), `leia_sr_d3d11.{cpp,h}`, `leia_sr_d3d12.{cpp,h}`, `leia_sr_gl.{cpp,h}` | | Background capture (transparency) | `leia_bg_capture_win.{cpp,h}` | -| 2D→3D lift (NeurD, D3D11) | `leia_lift_neurd.{cpp,h}`, `leia_neurd_abi.h` | +| 2D→3D lift (NeurD, D3D11) | `leia_lift_neurd.{cpp,h}` (+ private NeurD headers fetched at build time) | | Shaders | `shaders/` | ## Docs in this directory diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index 762ac5a..b427689 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -3,8 +3,8 @@ The Leia D3D11 display processor implements the runtime's **lift** slots — convert a single 2D RGBA frame into depth, a side-by-side stereo pair, or an N-view grid — by driving Leia's **NeurD** library. Source: `src/drv_leia/leia_lift_neurd.{h,cpp}` -(module), `src/drv_leia/leia_neurd_abi.h` (NeurD binary interface), and the slot glue -at the bottom of `src/drv_leia/leia_display_processor_d3d11.cpp`. +(module) and the slot glue at the bottom of `src/drv_leia/leia_display_processor_d3d11.cpp`, +built against the real NeurD headers fetched at build time (see *Building*). The runtime owns the lift *policy* — which content is lifted, the async worker thread, the result mailbox/ring, and how the output reaches the compositor. The plug-in owns @@ -38,10 +38,11 @@ the widest that fits D3D11's 16384-texel limit; a wider request fails with a WAR - **Dynamic load only.** `NeurD.dll` is found in NeurD's own loader order: `PATH` → `NEURD_PATH` (full path to the DLL) → `HKLM\SOFTWARE\LeiaInc\NeurD` (default value = - install dir), loaded with `LOAD_WITH_ALTERED_SEARCH_PATH`. No NeurD header, import - lib or DLL is needed to build; the binary interface is mirrored in - `leia_neurd_abi.h` (append-only function table, every entry version-gated before it - is read). + install dir), loaded with `LOAD_WITH_ALTERED_SEARCH_PATH`. No import lib: the module + resolves only the DLL's `NeurD_load` export and then calls through the returned + function table using the **real** `NeurD.h`'s header-inline wrappers and PFN types, + which version-gate every entry (an older runtime's table is shorter) — so any NeurD + signature drift is a compile error, not a runtime mismatch. - **Absent NeurD → nothing changes.** A presence probe (no `LoadLibrary`) runs on the first caps/stream call; if the DLL is not found the process state becomes *absent*, caps report `modes=0 / state=0`, and the plug-in behaves exactly as before. @@ -186,7 +187,33 @@ Read once per DP at create (`leia_lift_neurd_create`). Under the service, remember these are read by `displayxr-service.exe`'s environment, not the client's. -## Building against the runtime +## Building + +Lift compiles in only when BOTH of these hold; otherwise the four slots stay NULL, the +runtime reports lift unavailable, and the rest of the plug-in is unchanged. Neither +condition ever fails the plug-in build. + +### 1. The NeurD headers (private) + +`NeurD.h` and its closure (`_NeurD_detail.h`, `_NeurD_table.h`, and a generated +`NeurD_version.h`) are **Leia-private** and are never committed here. They are fetched +from the private **`LeiaInc/media_sdk`** repo at a pinned ref by +`scripts/fetch-neurd-headers.ps1` into the gitignored `NeurD-SDK-/include`, +exactly like the SR SDK: + +| | | +|---|---| +| Pins | `NEURD_SDK_REF` + `NEURD_SDK_REPO` in `scripts/build-windows.bat` **and** `.github/workflows/build-windows.yml` (`jobs.Build.env`), kept equal by `scripts/check_sr_pins.py` (lint.yml). | +| Current pin | `v0.4.6` — the newest media_sdk release tag; its headers are byte-identical to `dev@25a713d93` (the tip this module was written against). Move it to a newer **release tag** when the module needs a newer NeurD API. | +| Auth | `gh` with read access to `LeiaInc/media_sdk` (`gh auth login`, or `GH_TOKEN`). CI uses `secrets.LEIALOFT_GITHUB_TOKEN`, the SR SDK token — that token must be granted read on `media_sdk`. | +| Failure | Soft. The .bat prints a WARN and builds without lift; CI emits a `::warning::` annotation (`continue-on-error`) and ships without lift. CMake prints `NeurD headers NOT found ... lift compiled OUT`. | +| Override | `set NEURD_SDK_ROOT=` (e.g. a local media_sdk drop) skips the fetch. | + +`NeurD_version.h` is generated the way media_sdk's CMake does it, from +`sdk/NeurD_version.h.in` and the top-level `project(mediasdk VERSION x.y.z)` at the same +ref; it is the table version the module requests from `NeurD_load`. + +### 2. Runtime headers with the lift slots The slots compile only when the runtime headers define `XRT_DP_D3D11_HAS_LIFT`. The Windows pin `DXR_RUNTIME_GIT_TAG` is currently `v2.16.9`, which predates it — a pinned @@ -197,10 +224,17 @@ To build with lift before the runtime tags it, use a local runtime checkout — is committed: ```bat +:: gh must be able to read LeiaInc/media_sdk (and the SR SDK repo, as always) +gh auth status set DXR_RUNTIME_SOURCE_DIR=C:\dev\displayxr-runtime.wt-lift scripts\build-windows.bat build ``` +Confirm the configure output says `NeurD headers found ... enabling the D3D11 2D->3D lift +slots`, and at runtime the service log says `lift slots WIRED`. Set +`DXR_RUNTIME_SOURCE_DIR` explicitly: the script otherwise auto-detects a sibling +`displayxr-runtime` checkout, which may not carry the lift headers. + or `cmake -DDXR_RUNTIME_GIT_TAG=feat/lift-ext ...` for a fetched build. **Do not commit a branch or SHA pin**: `installer/CMakeLists.txt` derives `MIN_RUNTIME_VERSION` from `DXR_RUNTIME_GIT_TAG`, and the orchestrator/meta-installer assume a `vX.Y.Z` tag. Once diff --git a/scripts/build-windows.bat b/scripts/build-windows.bat index 5bd4c08..ac9b5be 100644 --- a/scripts/build-windows.bat +++ b/scripts/build-windows.bat @@ -14,6 +14,7 @@ setlocal enabledelayedexpansion :: :: Required environment (auto-fetched if missing): :: LEIASR_SDKROOT — extracted LeiaSR-SDK-*-win64 dir +:: NEURD_SDK_ROOT — dir with include\NeurD.h (+ generated NeurD_version.h); default: fetched :: :: Optional environment: :: DXR_RUNTIME_SOURCE_DIR — path to a local displayxr-runtime checkout. @@ -44,6 +45,17 @@ set SR_TAG=1.35.0.2011 :: Leia's PRIVATE artifact repo -- it is Leia SDK material and must not be :: publicly reachable. `gh auth status` must show an account with read access. set SR_SDK_REPO=LeiaInc/SR-SDK-Windows-Releases-Internal-Public +:: --- NeurD pins (2D->3D lift, docs/lift-neurd.md) ----------------------- +:: The lift module compiles against the REAL NeurD headers, fetched from the +:: PRIVATE LeiaInc/media_sdk repo at NEURD_SDK_REF into the gitignored +:: NeurD-SDK-\include (never committed -- Leia-private, like the SR SDK). +:: Same gh auth as the SR fetch. KEEP IN SYNC with build-windows.yml +:: (scripts/check_sr_pins.py asserts it). Pin a media_sdk RELEASE TAG; v0.4.6 +:: is the newest tag and its headers are byte-identical to dev@25a713d93 (the +:: tip the module was written against). A failed fetch is SOFT: lift compiles +:: out, nothing else changes. +set NEURD_SDK_REF=v0.4.6 +set NEURD_SDK_REPO=LeiaInc/media_sdk :: Stamp-aware (ST-5318) Vulkan weaver for SR 1.36.x. Separate tag because it is :: grafted from a different SR branch than SR_TAG's SDK — see the release notes. set SR_VKSTAMP_TAG=sr-sdk-v1.36.4.17537-vkstamp @@ -208,10 +220,26 @@ if not exist "%LEIASR_V2_SDKROOT%\lib\srSDK_loader.lib" ( echo SR v2 SDK ready. ) +:: --- NeurD headers (optional; soft failure) --- +:: An existing NEURD_SDK_ROOT (e.g. a local media_sdk\sdk drop with a generated +:: NeurD_version.h) takes priority; otherwise fetch the pinned headers. +if "%NEURD_SDK_ROOT%"=="" ( + set NEURD_SDK_ROOT=%REPO%NeurD-SDK-%NEURD_SDK_REF% +) +if not exist "%NEURD_SDK_ROOT%\include\NeurD_version.h" ( + echo === Fetching NeurD headers ^(%NEURD_SDK_REPO%@%NEURD_SDK_REF%^) === + powershell -NoProfile -ExecutionPolicy Bypass -File "%SCRIPT_DIR%fetch-neurd-headers.ps1" -Repo %NEURD_SDK_REPO% -Ref %NEURD_SDK_REF% -Dest "%NEURD_SDK_ROOT%" + if !ERRORLEVEL! NEQ 0 ( + echo WARN: NeurD headers unavailable -- building WITHOUT 2D->3D lift. + set NEURD_SDK_ROOT= + ) +) + echo. echo === Dependencies ready === echo LEIASR_SDKROOT=%LEIASR_SDKROOT% echo LEIASR_V2_SDKROOT=%LEIASR_V2_SDKROOT% +echo NEURD_SDK_ROOT=%NEURD_SDK_ROOT% echo DXR_RUNTIME_SOURCE_DIR=%DXR_RUNTIME_SOURCE_DIR% echo. @@ -256,6 +284,9 @@ REM because the two SDKs are independent drops and the migration needs both REM present at once. Unset simply builds the v1-only plug-in, which is the REM shipping configuration until a win64 v2 drop is generally available. REM set LEIASR_V2_SDKROOT=C:\path\to\LeiaSR-SDK-...-win64-Release +if not "%NEURD_SDK_ROOT%"=="" ( + set CMAKE_ARGS=!CMAKE_ARGS! -DNEURD_SDK_ROOT="%NEURD_SDK_ROOT%" +) if not "%LEIASR_V2_SDKROOT%"=="" ( set CMAKE_ARGS=!CMAKE_ARGS! -DLEIASR_V2_SDKROOT="%LEIASR_V2_SDKROOT%" echo Building WITH the SR v2 C API ^(LEIASR_V2_SDKROOT set^). diff --git a/scripts/check_sr_pins.py b/scripts/check_sr_pins.py index 4016cb4..5df8694 100644 --- a/scripts/check_sr_pins.py +++ b/scripts/check_sr_pins.py @@ -67,7 +67,10 @@ LINUX_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build-linux.yml" # The five pins that must be identical in the .bat and in build-windows.yml. -PINS = ("SR_TAG", "SR_VKSTAMP_TAG", "SR_V2_TAG", "SR_V2_DIR", "SR_SDK_REPO") +# NEURD_SDK_REF / NEURD_SDK_REPO pin the private NeurD headers the 2D->3D lift +# module is built against (docs/lift-neurd.md) -- same two-file duplication, +# same drift hazard, so they ride the same check. +PINS = ("SR_TAG", "SR_VKSTAMP_TAG", "SR_V2_TAG", "SR_V2_DIR", "SR_SDK_REPO", "NEURD_SDK_REF", "NEURD_SDK_REPO") # sr-sdk-v2-1.37.0.1502 -> ("1.37.0", "1502") SR_V2_TAG_RE = re.compile(r"^sr-sdk-v2-(\d+\.\d+\.\d+)\.(\d+)$") diff --git a/scripts/fetch-neurd-headers.ps1 b/scripts/fetch-neurd-headers.ps1 new file mode 100644 index 0000000..f4df3ca --- /dev/null +++ b/scripts/fetch-neurd-headers.ps1 @@ -0,0 +1,64 @@ +# Copyright 2026, Leia Inc. +# SPDX-License-Identifier: Apache-2.0 +# +# Fetch the NeurD public headers from the PRIVATE LeiaInc/media_sdk repo at a +# pinned ref, for the plug-in's 2D->3D lift module (docs/lift-neurd.md). +# +# These headers are Leia-private: they land in a gitignored directory +# (NeurD-SDK-/include, see .gitignore) and must never be committed or +# re-hosted on this public repo -- same rule as the SR SDK. +# +# Auth: `gh` with an account (or GH_TOKEN) that can read LeiaInc/media_sdk. +# CI passes secrets.LEIALOFT_GITHUB_TOKEN, the token the SR SDK fetch uses. +# +# Exit 0 = headers ready; 1 = fetch failed. Callers treat failure as SOFT: +# the lift module compiles out and the rest of the plug-in builds unchanged. +# +# Usage: fetch-neurd-headers.ps1 -Repo LeiaInc/media_sdk -Ref v0.4.6 -Dest + +param( + [Parameter(Mandatory = $true)][string]$Repo, + [Parameter(Mandatory = $true)][string]$Ref, + [Parameter(Mandatory = $true)][string]$Dest +) + +$ErrorActionPreference = 'Stop' +$inc = Join-Path $Dest 'include' + +function Get-RepoFile([string]$path) { + # Raw file contents at the pinned ref. gh exits non-zero on 404 / no auth. + $out = & gh api "repos/$Repo/contents/${path}?ref=$Ref" -H 'Accept: application/vnd.github.raw' 2>&1 + if ($LASTEXITCODE -ne 0) { throw "gh api $path@$Ref failed: $out" } + return ($out -join "`n") + "`n" +} + +try { + New-Item -ItemType Directory -Force $inc | Out-Null + + # NeurD.h includes NeurD_version.h and _NeurD_detail.h; the latter includes + # _NeurD_table.h. That is the whole closure. + foreach ($f in @('NeurD.h', '_NeurD_detail.h', '_NeurD_table.h')) { + Set-Content -Path (Join-Path $inc $f) -Value (Get-RepoFile "sdk/$f") -NoNewline -Encoding UTF8 + } + + # NeurD_version.h is generated by media_sdk's CMake from NeurD_version.h.in + # and the top-level project(... VERSION "x.y.z"). Reproduce that exactly. + $top = Get-RepoFile 'CMakeLists.txt' + $m = [regex]::Match($top, 'project\s*\(\s*mediasdk[^)]*VERSION\s+"?(\d+)\.(\d+)\.(\d+)"?') + if (-not $m.Success) { throw "could not find project(mediasdk ... VERSION x.y.z) in CMakeLists.txt@$Ref" } + $ver = (Get-RepoFile 'sdk/NeurD_version.h.in'). + Replace('@PROJECT_VERSION_MAJOR@', $m.Groups[1].Value). + Replace('@PROJECT_VERSION_MINOR@', $m.Groups[2].Value). + Replace('@PROJECT_VERSION_PATCH@', $m.Groups[3].Value) + Set-Content -Path (Join-Path $inc 'NeurD_version.h') -Value $ver -NoNewline -Encoding UTF8 + + Write-Host "NeurD headers $($m.Groups[1].Value).$($m.Groups[2].Value).$($m.Groups[3].Value) ($Repo@$Ref) -> $inc" + exit 0 +} catch { + Write-Host "WARNING: NeurD header fetch failed: $_" + Write-Host " 2D->3D lift will be compiled OUT (plug-in otherwise unchanged)." + Write-Host " Needs gh auth with read access to $Repo (gh auth login / GH_TOKEN)." + # Do not leave a half-populated include dir that CMake would accept. + if (Test-Path $inc) { Remove-Item -Recurse -Force $inc } + exit 1 +} diff --git a/src/drv_leia/CMakeLists.txt b/src/drv_leia/CMakeLists.txt index 7147563..8458b8b 100644 --- a/src/drv_leia/CMakeLists.txt +++ b/src/drv_leia/CMakeLists.txt @@ -183,10 +183,6 @@ if(EXISTS "${SR_PATH}/lib/SimulatedRealityDirectX.lib") list(APPEND LEIA_SOURCES leia_sr_d3d11.cpp leia_sr_d3d11.h leia_display_processor_d3d11.cpp leia_display_processor_d3d11.h - # 2D->3D lift via NeurD (docs/lift-neurd.md). NeurD.dll is loaded - # dynamically at first use — no NeurD headers, libs or DLLs are needed - # to build, and a box without NeurD loads the plug-in unchanged. - leia_lift_neurd.cpp leia_lift_neurd.h leia_neurd_abi.h leia_sr_d3d12.cpp leia_sr_d3d12.h leia_display_processor_d3d12.cpp leia_display_processor_d3d12.h) set(LEIA_SR_D3D11_AVAILABLE TRUE) @@ -202,6 +198,33 @@ if(EXISTS "${SR_PATH}/lib/SimulatedRealityOpenGL.lib") set(LEIA_SR_GL_AVAILABLE TRUE) endif() +# --- NeurD 2D->3D lift (optional, additive) --------------------------------- +# docs/lift-neurd.md. The module compiles against the REAL NeurD headers, which +# are Leia-private and are never committed here: scripts/build-windows.bat and +# build-windows.yml fetch them from LeiaInc/media_sdk at NEURD_SDK_REF (pinned +# next to the SR pins) into a gitignored NeurD-SDK-/include. NeurD.dll is +# still loaded dynamically at first use — no import lib, no link dependency. +# Headers absent (no token / offline / fork PR) => the module compiles OUT, the +# DP's lift slots stay NULL, the runtime reports lift unavailable, and the rest +# of the plug-in builds exactly as before. Never a hard failure. +set(LEIA_NEURD_AVAILABLE FALSE) +if(NOT DEFINED NEURD_SDK_ROOT AND DEFINED ENV{NEURD_SDK_ROOT}) + set(NEURD_SDK_ROOT "$ENV{NEURD_SDK_ROOT}") +endif() +if(LEIA_SR_D3D11_AVAILABLE AND NEURD_SDK_ROOT + AND EXISTS "${NEURD_SDK_ROOT}/include/NeurD.h" + AND EXISTS "${NEURD_SDK_ROOT}/include/NeurD_version.h" + AND EXISTS "${NEURD_SDK_ROOT}/include/_NeurD_detail.h" + AND EXISTS "${NEURD_SDK_ROOT}/include/_NeurD_table.h") + set(LEIA_NEURD_AVAILABLE TRUE) + list(APPEND LEIA_SOURCES leia_lift_neurd.cpp leia_lift_neurd.h) + message(STATUS "NeurD headers found at ${NEURD_SDK_ROOT} - enabling the D3D11 2D->3D lift slots") +elseif(LEIA_SR_D3D11_AVAILABLE) + message(STATUS "NeurD headers NOT found (NEURD_SDK_ROOT='${NEURD_SDK_ROOT}') - 2D->3D lift compiled OUT; " + "the plug-in is otherwise unchanged. Fetch them with scripts/build-windows.bat " + "(needs gh auth with read access to LeiaInc/media_sdk).") +endif() + # Windows Graphics Capture for compose-under-bg transparency. The GL DP imports # the captured D3D11 desktop texture into GL via WGL_NV_DX_interop2, so it needs # this module too. @@ -344,6 +367,11 @@ if(LEIA_SR_D3D11_AVAILABLE) target_compile_definitions(DisplayXR-LeiaSR PRIVATE XRT_HAVE_LEIA_SR_D3D11) endif() +if(LEIA_NEURD_AVAILABLE) + target_include_directories(DisplayXR-LeiaSR PRIVATE "${NEURD_SDK_ROOT}/include") + target_compile_definitions(DisplayXR-LeiaSR PRIVATE DXR_LEIA_HAS_NEURD) +endif() + if(LEIA_SR_D3D12_AVAILABLE) target_link_libraries(DisplayXR-LeiaSR PRIVATE d3d12 d3dcompiler) target_compile_definitions(DisplayXR-LeiaSR PRIVATE XRT_HAVE_LEIA_SR_D3D12) diff --git a/src/drv_leia/leia_display_processor_d3d11.cpp b/src/drv_leia/leia_display_processor_d3d11.cpp index fb1a319..f0191df 100644 --- a/src/drv_leia/leia_display_processor_d3d11.cpp +++ b/src/drv_leia/leia_display_processor_d3d11.cpp @@ -56,11 +56,13 @@ * 2D->3D lift (runtime feat/lift-ext, XRT_DP_D3D11_HAS_LIFT) — four appended * slots backed by NeurD (leia_lift_neurd.cpp, docs/lift-neurd.md). Same * coupled-addition pattern: against older runtime headers the slots compile out - * and struct_size tells the runtime they are absent. The NeurD module itself is - * runtime-header-independent and always built; it never loads NeurD.dll until - * a lift slot is actually called. + * and struct_size tells the runtime they are absent. The NeurD module also + * needs the real NeurD headers, fetched at build time from the private + * media_sdk repo (DXR_LEIA_HAS_NEURD); without them the slots stay NULL and the + * runtime reports lift unavailable. Even when built, NeurD.dll is loaded only + * when a lift slot is first called. */ -#if defined(XRT_DP_D3D11_HAS_LIFT) +#if defined(XRT_DP_D3D11_HAS_LIFT) && defined(DXR_LEIA_HAS_NEURD) #define DXR_LEIA_DP_D3D11_LIFT 1 #include "leia_lift_neurd.h" #include // offsetof @@ -2566,7 +2568,11 @@ leia_dp_d3d11_init_vtable(struct leia_display_processor_d3d11_impl *ldp) ldp->lift = leia_lift_neurd_create(); // reads DXR_LEIA_LIFT* once; loads nothing U_LOG_W("Leia D3D11 DP: lift slots WIRED (NeurD, loaded on first use)"); #else +#if !defined(XRT_DP_D3D11_HAS_LIFT) U_LOG_W("Leia D3D11 DP: lift slots NOT COMPILED (runtime headers predate XRT_DP_D3D11_HAS_LIFT)"); +#else + U_LOG_W("Leia D3D11 DP: lift slots NOT COMPILED (NeurD headers were not fetched at build time)"); +#endif #endif } diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index 290f463..eb19529 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -47,7 +47,13 @@ */ #include "leia_lift_neurd.h" -#include "leia_neurd_abi.h" +// The REAL NeurD header, fetched at build time from the private LeiaInc/media_sdk +// repo at the pinned NEURD_SDK_REF (never committed here; see docs/lift-neurd.md). +// Only its types, PFN typedefs and header-inline table wrappers are used — the +// wrappers dispatch through the table NeurD_load returns and version-check each +// entry — so there is no import lib and no link dependency on NeurD, and any +// NeurD signature drift is a compile error in this file. +#include #include "util/u_logging.h" @@ -68,6 +74,11 @@ #include #include +//! Table entry @p name is present in the LOADED runtime (version test first, so +//! an older, physically shorter table is never read past its end). The +//! NEURD__INTRODUCED_IN constants come from the real header. +#define LEIA_NEURD_HAS(nd, name) ((nd) != nullptr && (nd)->version >= NEURD_##name##_INTRODUCED_IN && (nd)->name != nullptr) + namespace { /* @@ -124,32 +135,32 @@ clampf(float v, float lo, float hi) } const char * -status_str(enum leia_neurd_status s) +status_str(enum NeurD_status s) { switch (s) { - case LEIA_NEURD_SUCCESS: return "SUCCESS"; - case LEIA_NEURD_IN_PROGRESS: return "IN_PROGRESS"; - case LEIA_NEURD_GENERIC_ERROR: return "GENERIC_ERROR"; - case LEIA_NEURD_INVALID_ARG: return "INVALID_ARG"; - case LEIA_NEURD_INVALID_LICENSE: return "INVALID_LICENSE"; - case LEIA_NEURD_UNAVAILABLE_OUTDATED_RUNTIME: return "UNAVAILABLE_OUTDATED_RUNTIME"; - case LEIA_NEURD_NOT_INITIALIZED: return "NOT_INITIALIZED"; - case LEIA_NEURD_LICENSE_NETWORK_ERROR: return "LICENSE_NETWORK_ERROR"; - case LEIA_NEURD_INVALID_MODEL: return "INVALID_MODEL"; - case LEIA_NEURD_FILESYSTEM_ERROR: return "FILESYSTEM_ERROR"; - case LEIA_NEURD_NETWORK_ERROR: return "NETWORK_ERROR"; - case LEIA_NEURD_ABORTED: return "ABORTED"; + case NEURD_SUCCESS: return "SUCCESS"; + case NEURD_IN_PROGRESS: return "IN_PROGRESS"; + case NEURD_GENERIC_ERROR: return "GENERIC_ERROR"; + case NEURD_INVALID_ARG: return "INVALID_ARG"; + case NEURD_INVALID_LICENSE: return "INVALID_LICENSE"; + case NEURD_UNAVAILABLE_OUTDATED_RUNTIME: return "UNAVAILABLE_OUTDATED_RUNTIME"; + case NEURD_NOT_INITIALIZED: return "NOT_INITIALIZED"; + case NEURD_LICENSE_NETWORK_ERROR: return "LICENSE_NETWORK_ERROR"; + case NEURD_INVALID_MODEL: return "INVALID_MODEL"; + case NEURD_FILESYSTEM_ERROR: return "FILESYSTEM_ERROR"; + case NEURD_NETWORK_ERROR: return "NETWORK_ERROR"; + case NEURD_ABORTED: return "ABORTED"; default: return "UNKNOWN"; } } const char * -backend_str(enum leia_neurd_backend b) +backend_str(enum NeurD_backend b) { switch (b) { - case LEIA_NEURD_BACKEND_CUDA: return "cuda"; - case LEIA_NEURD_BACKEND_DIRECTML: return "directml"; - case LEIA_NEURD_BACKEND_OPENVINO: return "openvino"; + case NEURD_BACKEND_CUDA: return "cuda"; + case NEURD_BACKEND_DIRECTML: return "directml"; + case NEURD_BACKEND_OPENVINO: return "openvino"; default: return "unknown"; } } @@ -163,9 +174,9 @@ backend_str(enum leia_neurd_backend b) enum backend_choice { BACKEND_AUTO = -1, - BACKEND_CUDA = LEIA_NEURD_BACKEND_CUDA, - BACKEND_DIRECTML = LEIA_NEURD_BACKEND_DIRECTML, - BACKEND_OPENVINO = LEIA_NEURD_BACKEND_OPENVINO, + BACKEND_CUDA = NEURD_BACKEND_CUDA, + BACKEND_DIRECTML = NEURD_BACKEND_DIRECTML, + BACKEND_OPENVINO = NEURD_BACKEND_OPENVINO, }; struct knobs @@ -189,7 +200,7 @@ read_knobs() struct knobs k = {}; k.enabled = true; k.backend = BACKEND_DIRECTML; - k.autoscaling = LEIA_NEURD_AUTOSCALING_720P; + k.autoscaling = NEURD_INPUT_AUTOSCALING_720P; k.view_gain = 1.0f; const char *e = std::getenv("DXR_LEIA_LIFT"); @@ -218,13 +229,13 @@ read_knobs() if (e != nullptr && e[0] != '\0') { k.scale_forced = true; if (env_ieq(e, "none") || env_ieq(e, "native") || env_ieq(e, "0")) { - k.autoscaling = LEIA_NEURD_AUTOSCALING_NONE; + k.autoscaling = NEURD_INPUT_AUTOSCALING_NONE; } else if (env_ieq(e, "720") || env_ieq(e, "720p")) { - k.autoscaling = LEIA_NEURD_AUTOSCALING_720P; + k.autoscaling = NEURD_INPUT_AUTOSCALING_720P; } else if (env_ieq(e, "1080") || env_ieq(e, "1080p")) { - k.autoscaling = LEIA_NEURD_AUTOSCALING_1080P; + k.autoscaling = NEURD_INPUT_AUTOSCALING_1080P; } else if (env_ieq(e, "1440") || env_ieq(e, "1440p")) { - k.autoscaling = LEIA_NEURD_AUTOSCALING_1440P; + k.autoscaling = NEURD_INPUT_AUTOSCALING_1440P; } else { U_LOG_W("Leia lift: DXR_LEIA_LIFT_SCALE='%s' not recognised (720|1080|1440|none) — using 720", e); } @@ -318,11 +329,11 @@ struct global std::atomic streams_live{0}; int requested_backend = BACKEND_DIRECTML; //!< First acquirer's choice (NeurD's forced backend is sticky). - int32_t default_autoscaling = LEIA_NEURD_AUTOSCALING_720P; + int32_t default_autoscaling = NEURD_INPUT_AUTOSCALING_720P; HMODULE lib = nullptr; - const struct leia_neurd_table *nd = nullptr; - enum leia_neurd_backend backend = LEIA_NEURD_BACKEND_DIRECTML; + struct NeurD const *nd = nullptr; + enum NeurD_backend backend = NEURD_BACKEND_DIRECTML; char backend_name[32] = {}; ID3D11Device *nd_dev = nullptr; //!< NeurD-owned: never Released. @@ -503,7 +514,7 @@ compile_cs(ID3D11Device *dev, const char *src, const char *name) bool setup_nd_device_locked() { - g.nd_dev = static_cast(g.nd->get_dx_device()); + g.nd_dev = static_cast(NeurD_get_dx_device(g.nd)); if (g.nd_dev == nullptr) { U_LOG_W("Leia lift: NeurD backend '%s' exposes no D3D11 device — the D3D11 lift path needs the " "DirectML backend (DXR_LEIA_LIFT_BACKEND=directml). Lift unavailable.", @@ -563,9 +574,9 @@ activation_worker() g.worker_running.store(false); return; } - auto load = reinterpret_cast(reinterpret_cast(GetProcAddress(lib, "NeurD_load"))); - struct leia_neurd_load_request req = {LEIA_NEURD_BUILT_AGAINST}; - const struct leia_neurd_table *nd = load != nullptr ? load(&req) : nullptr; + auto load = reinterpret_cast(reinterpret_cast(GetProcAddress(lib, "NeurD_load"))); + struct NeurD_load_request req = {NEURD_VERSION}; + struct NeurD const *nd = load != nullptr ? load(&req) : nullptr; if (nd == nullptr) { U_LOG_W("Leia lift: NeurD.dll (%s) has no usable NeurD_load — lift unavailable", where); // Deliberately no FreeLibrary: NeurD may have spun threads. @@ -574,15 +585,15 @@ activation_worker() return; } U_LOG_W("Leia lift: loaded NeurD %u.%u.%u from %s (plug-in built against %u.%u.%u)", - LEIA_NEURD_VERSION_MAJOR(nd->version), LEIA_NEURD_VERSION_MINOR(nd->version), - LEIA_NEURD_VERSION_PATCH(nd->version), where, - LEIA_NEURD_VERSION_MAJOR(LEIA_NEURD_BUILT_AGAINST), - LEIA_NEURD_VERSION_MINOR(LEIA_NEURD_BUILT_AGAINST), - LEIA_NEURD_VERSION_PATCH(LEIA_NEURD_BUILT_AGAINST)); - if (LEIA_NEURD_VERSION_MAJOR(nd->version) != 0 || - !LEIA_NEURD_HAS(nd, convert_stream_dx, 0, 3, 11) || !LEIA_NEURD_HAS(nd, get_dx_device, 0, 4, 3) || - !LEIA_NEURD_HAS(nd, create_stream, 0, 3, 11) || !LEIA_NEURD_HAS(nd, set_prop_1i, 0, 2, 1) || - !LEIA_NEURD_HAS(nd, set_prop_1f, 0, 2, 1)) { + NEURD_GET_VERSION_MAJOR(nd->version), NEURD_GET_VERSION_MINOR(nd->version), + NEURD_GET_VERSION_PATCH(nd->version), where, + NEURD_GET_VERSION_MAJOR(NEURD_VERSION), + NEURD_GET_VERSION_MINOR(NEURD_VERSION), + NEURD_GET_VERSION_PATCH(NEURD_VERSION)); + if (NEURD_GET_VERSION_MAJOR(nd->version) != 0 || + !LEIA_NEURD_HAS(nd, convert_stream_dx) || !LEIA_NEURD_HAS(nd, get_dx_device) || + !LEIA_NEURD_HAS(nd, create_stream) || !LEIA_NEURD_HAS(nd, set_prop_1i) || + !LEIA_NEURD_HAS(nd, set_prop_1f)) { U_LOG_W("Leia lift: NeurD runtime too old/new for the D3D11 stream path (need 0.4.3+, " "major 0) — lift unavailable"); g.state.store(G_FAILED); @@ -591,43 +602,43 @@ activation_worker() } g.lib = lib; g.nd = nd; - if (LEIA_NEURD_HAS(nd, set_logger_callback, 0, 2, 5)) { - nd->set_logger_callback(neurd_log_cb); + if (LEIA_NEURD_HAS(nd, set_logger_callback)) { + NeurD_set_logger_callback(nd, neurd_log_cb); } } - const struct leia_neurd_table *nd = g.nd; + struct NeurD const *nd = g.nd; // Properties that must precede init (NeurD reads some at init time): // the defaults this plug-in wants for every stream. - nd->set_prop_1i(LEIA_NEURD_PROP_OUTPUT_TYPE, LEIA_NEURD_OUTPUT_SBS); - nd->set_prop_1i(LEIA_NEURD_PROP_INPUT_AUTOSCALING, g.default_autoscaling); - nd->set_prop_1i(LEIA_NEURD_PROP_INPAINT_TYPE, LEIA_NEURD_INPAINT_V1_STRETCH); + NeurD_set_prop_1i(nd, NEURD_PROP_OUTPUT_TYPE, NEURD_OUTPUT_IMAGE_TYPE_SBS); + NeurD_set_prop_1i(nd, NEURD_PROP_INPUT_AUTOSCALING, g.default_autoscaling); + NeurD_set_prop_1i(nd, NEURD_PROP_INPAINT_TYPE, NEURD_INPAINT_TYPE_V1_STRETCH); // Backend must be forced immediately before init. NeurD keeps a forced // backend across deinit, so this is effectively set once per process. - if (g.requested_backend != BACKEND_AUTO && LEIA_NEURD_HAS(nd, set_backend, 0, 3, 4)) { - enum leia_neurd_status bs = nd->set_backend((enum leia_neurd_backend)g.requested_backend); - if (bs != LEIA_NEURD_SUCCESS) { - U_LOG_W("Leia lift: NeurD_set_backend(%s) -> %s", backend_str((enum leia_neurd_backend)g.requested_backend), + if (g.requested_backend != BACKEND_AUTO && LEIA_NEURD_HAS(nd, set_backend)) { + enum NeurD_status bs = NeurD_set_backend(nd, (enum NeurD_backend)g.requested_backend); + if (bs != NEURD_SUCCESS) { + U_LOG_W("Leia lift: NeurD_set_backend(%s) -> %s", backend_str((enum NeurD_backend)g.requested_backend), status_str(bs)); } } - enum leia_neurd_status st; - if (LEIA_NEURD_HAS(nd, init_with_options, 0, 4, 6)) { - struct leia_neurd_init_options opts = {}; + enum NeurD_status st; + if (LEIA_NEURD_HAS(nd, init_with_options)) { + struct NeurD_init_options opts = {}; opts.struct_size = sizeof(opts); - opts.photo_model_id = LEIA_NEURD_MODEL_PHOTO_RELATIVE_QUALITY; - opts.video_model_id = LEIA_NEURD_MODEL_VIDEO_RELATIVE_FAST; - st = nd->init_with_options(1 /* multithreaded: we call from the runtime's lift thread */, &opts); - } else if (LEIA_NEURD_HAS(nd, init, 0, 2, 2)) { - st = nd->init(1); + opts.photo_model_id = NEURD_MODEL_PHOTO_RELATIVE_QUALITY; + opts.video_model_id = NEURD_MODEL_VIDEO_RELATIVE_FAST; + st = NeurD_init_with_options(nd, 1 /* multithreaded: we call from the runtime's lift thread */, &opts); + } else if (LEIA_NEURD_HAS(nd, init)) { + st = NeurD_init(nd, 1); } else { - st = LEIA_NEURD_UNAVAILABLE_OUTDATED_RUNTIME; + st = NEURD_UNAVAILABLE_OUTDATED_RUNTIME; } - if (st == LEIA_NEURD_LICENSE_NETWORK_ERROR || st == LEIA_NEURD_NETWORK_ERROR) { + if (st == NEURD_LICENSE_NETWORK_ERROR || st == NEURD_NETWORK_ERROR) { g.next_retry_ms.store(now_ms() + kActivationRetryMs); U_LOG_W("Leia lift: NeurD licence activation needs the network (%s) — reporting ACTIVATING, " "retrying in %llus", @@ -636,24 +647,24 @@ activation_worker() g.worker_running.store(false); return; } - if (st != LEIA_NEURD_SUCCESS) { + if (st != NEURD_SUCCESS) { U_LOG_W("Leia lift: NeurD init failed: %s — lift unavailable", status_str(st)); g.state.store(G_FAILED); g.worker_running.store(false); return; } - enum leia_neurd_backend be = (g.requested_backend == BACKEND_AUTO) - ? LEIA_NEURD_BACKEND_DIRECTML - : (enum leia_neurd_backend)g.requested_backend; - if (LEIA_NEURD_HAS(nd, get_backend, 0, 3, 4)) { - nd->get_backend(&be); + enum NeurD_backend be = (g.requested_backend == BACKEND_AUTO) + ? NEURD_BACKEND_DIRECTML + : (enum NeurD_backend)g.requested_backend; + if (LEIA_NEURD_HAS(nd, get_backend)) { + NeurD_get_backend(nd, &be); } std::lock_guard lock(g.mtx); g.backend = be; snprintf(g.backend_name, sizeof(g.backend_name), "neurd-%s", backend_str(be)); - g.interactive_unavailable = !LEIA_NEURD_HAS(nd, convert_stream_dx_interactive, 0, 4, 5); + g.interactive_unavailable = !LEIA_NEURD_HAS(nd, convert_stream_dx_interactive); g.props_valid = false; next = setup_nd_device_locked() ? (uint32_t)G_READY : (uint32_t)G_FAILED; if (next == G_READY) { @@ -734,7 +745,7 @@ struct lift_stream uint32_t mode; uint32_t content_hint; float input_scale; - struct leia_neurd_stream *ns; //!< Created lazily on first convert (NeurD must be READY). + struct NeurD_stream *ns; //!< Created lazily on first convert (NeurD must be READY). ID3D11Device *dev; //!< Caller's device (AddRef'd) the bridges are opened on. ID3D11Query *our_done; @@ -789,8 +800,8 @@ release_stream_locked(lift_stream *s) release_out_bridge(s); safe_release(s->our_done); safe_release(s->dev); - if (s->ns != nullptr && g.state.load() == G_READY && LEIA_NEURD_HAS(g.nd, destroy_stream, 0, 3, 11)) { - g.nd->destroy_stream(s->ns); + if (s->ns != nullptr && g.state.load() == G_READY && LEIA_NEURD_HAS(g.nd, destroy_stream)) { + NeurD_destroy_stream(g.nd, s->ns); } s->ns = nullptr; } @@ -954,7 +965,7 @@ groups16(uint32_t n) //! Copy/unpack NeurD's output into the stream's output bridge (NeurD device). bool -stage_output(lift_stream *s, const struct leia_neurd_image &out) +stage_output(lift_stream *s, const struct NeurD_image &out) { auto *res = static_cast(out.data); if (res == nullptr || out.width <= 0 || out.height <= 0) { @@ -1027,13 +1038,13 @@ stage_output(lift_stream *s, const struct leia_neurd_image &out) //! Apply one global NeurD property only if it changed. Called with g.mtx held. bool -prop_i(enum leia_neurd_prop p, int32_t v, int32_t &cache) +prop_i(enum NeurD_prop p, int32_t v, int32_t &cache) { if (g.props_valid && cache == v) { return true; } - enum leia_neurd_status st = g.nd->set_prop_1i(p, v); - if (st != LEIA_NEURD_SUCCESS) { + enum NeurD_status st = NeurD_set_prop_1i(g.nd, p, v); + if (st != NEURD_SUCCESS) { U_LOG_W("Leia lift: NeurD_set_prop_1i(%d, %d) -> %s", (int)p, (int)v, status_str(st)); cache = -1; return false; @@ -1043,13 +1054,13 @@ prop_i(enum leia_neurd_prop p, int32_t v, int32_t &cache) } bool -prop_f(enum leia_neurd_prop p, float v, float &cache) +prop_f(enum NeurD_prop p, float v, float &cache) { if (g.props_valid && cache == v) { return true; } - enum leia_neurd_status st = g.nd->set_prop_1f(p, v); - if (st != LEIA_NEURD_SUCCESS) { + enum NeurD_status st = NeurD_set_prop_1f(g.nd, p, v); + if (st != NEURD_SUCCESS) { U_LOG_W("Leia lift: NeurD_set_prop_1f(%d, %f) -> %s", (int)p, (double)v, status_str(st)); cache = -1.0f; return false; @@ -1070,15 +1081,15 @@ autoscale_for(const lift_stream *s, const struct knobs &k, uint32_t in_h) // Smallest NeurD bucket that still covers the requested inference height. float target = s->input_scale * (float)in_h; if (target <= 720.0f) { - return LEIA_NEURD_AUTOSCALING_720P; + return NEURD_INPUT_AUTOSCALING_720P; } if (target <= 1080.0f) { - return LEIA_NEURD_AUTOSCALING_1080P; + return NEURD_INPUT_AUTOSCALING_1080P; } if (target <= 1440.0f) { - return LEIA_NEURD_AUTOSCALING_1440P; + return NEURD_INPUT_AUTOSCALING_1440P; } - return LEIA_NEURD_AUTOSCALING_NONE; + return NEURD_INPUT_AUTOSCALING_NONE; } } // namespace @@ -1139,10 +1150,10 @@ leia_lift_neurd_destroy(struct leia_lift_neurd **plift) l->streams.clear(); if (l->acquired) { l->acquired = false; - if (--g.refcount == 0 && g.state.load() == G_READY && LEIA_NEURD_HAS(g.nd, shrink_memory_pool, 0, 3, 10)) { + if (--g.refcount == 0 && g.state.load() == G_READY && LEIA_NEURD_HAS(g.nd, shrink_memory_pool)) { // Last user gone: give NeurD's pools back but stay initialised // (re-init = licence + model load; DPs are recreated on focus change). - g.nd->shrink_memory_pool(); + NeurD_shrink_memory_pool(g.nd); g.props_valid = false; } } @@ -1178,13 +1189,13 @@ leia_lift_neurd_get_caps(struct leia_lift_neurd *l, struct leia_lift_neurd_caps // behind a convert holding g.mtx. snprintf(out->backend, sizeof(out->backend), "%s", g.backend_name); if (lat == 0) { - lat = (g.backend == LEIA_NEURD_BACKEND_DIRECTML) ? kPriorLatencyDirectMlNs - : (g.backend == LEIA_NEURD_BACKEND_CUDA) ? kPriorLatencyCudaNs + lat = (g.backend == NEURD_BACKEND_DIRECTML) ? kPriorLatencyDirectMlNs + : (g.backend == NEURD_BACKEND_CUDA) ? kPriorLatencyCudaNs : kPriorLatencyOtherNs; } } else { snprintf(out->backend, sizeof(out->backend), "neurd-%s", - l->k.backend == BACKEND_AUTO ? "auto" : backend_str((enum leia_neurd_backend)l->k.backend)); + l->k.backend == BACKEND_AUTO ? "auto" : backend_str((enum NeurD_backend)l->k.backend)); if (lat == 0) { lat = kPriorLatencyDirectMlNs; } @@ -1286,7 +1297,7 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, try { std::lock_guard lock(g.mtx); - const struct leia_neurd_table *nd = g.nd; + struct NeurD const *nd = g.nd; lift_stream *s = nullptr; for (lift_stream *it : l->streams) { @@ -1360,8 +1371,8 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, // ---- Lazily create the NeurD stream. if (s->ns == nullptr) { - enum leia_neurd_status cs = nd->create_stream(&s->ns); - if (cs != LEIA_NEURD_SUCCESS || s->ns == nullptr) { + enum NeurD_status cs = NeurD_create_stream(nd, &s->ns); + if (cs != NEURD_SUCCESS || s->ns == nullptr) { in_tex->Release(); s->ns = nullptr; LIFT_WARN_ONCE("Leia lift: NeurD_create_stream -> %s", status_str(cs)); @@ -1403,9 +1414,9 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, } uint32_t views = 2; uint32_t cols = 2, rows = 1; - int32_t out_type = LEIA_NEURD_OUTPUT_SBS; + int32_t out_type = NEURD_OUTPUT_IMAGE_TYPE_SBS; if (s->mode == LEIA_LIFT_MODE_DEPTH) { - out_type = LEIA_NEURD_OUTPUT_DEPTH; + out_type = NEURD_OUTPUT_IMAGE_TYPE_DEPTH; views = 1; cols = rows = 1; } else if (s->mode == LEIA_LIFT_MODE_NVIEW) { @@ -1419,18 +1430,18 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, const float gain = (dp.strength >= 0.0f) ? clampf(dp.strength, 0.0f, 10.0f) : 1.0f; // NeurD always fills disocclusions; non-zero picks the blur fill, 0 the // cheaper edge stretch. - const int32_t inpaint = (dp.inpaint != 0) ? LEIA_NEURD_INPAINT_V1_BLUR : LEIA_NEURD_INPAINT_V1_STRETCH; + const int32_t inpaint = (dp.inpaint != 0) ? NEURD_INPAINT_TYPE_V1_BLUR : NEURD_INPAINT_TYPE_V1_STRETCH; - bool props_ok = prop_i(LEIA_NEURD_PROP_OUTPUT_TYPE, out_type, g.p_out_type) && - prop_i(LEIA_NEURD_PROP_OUTPUT_TILES_W, (int32_t)cols, g.p_tiles_w) && - prop_i(LEIA_NEURD_PROP_OUTPUT_TILES_H, (int32_t)rows, g.p_tiles_h) && - prop_i(LEIA_NEURD_PROP_INPAINT_TYPE, inpaint, g.p_inpaint) && - prop_i(LEIA_NEURD_PROP_INPUT_AUTOSCALING, autoscale_for(s, l->k, h), + bool props_ok = prop_i(NEURD_PROP_OUTPUT_TYPE, out_type, g.p_out_type) && + prop_i(NEURD_PROP_OUTPUT_TILES_W, (int32_t)cols, g.p_tiles_w) && + prop_i(NEURD_PROP_OUTPUT_TILES_H, (int32_t)rows, g.p_tiles_h) && + prop_i(NEURD_PROP_INPAINT_TYPE, inpaint, g.p_inpaint) && + prop_i(NEURD_PROP_INPUT_AUTOSCALING, autoscale_for(s, l->k, h), g.p_autoscale) && - prop_i(LEIA_NEURD_PROP_AUTO_CONVERGENCE, auto_conv ? 1 : 0, g.p_autoconv) && - prop_f(LEIA_NEURD_PROP_GAIN_MULTIPLIER, gain, g.p_gain); + prop_i(NEURD_PROP_AUTO_CONVERGENCE, auto_conv ? 1 : 0, g.p_autoconv) && + prop_f(NEURD_PROP_GAIN_MULTIPLIER, gain, g.p_gain); if (props_ok && !auto_conv) { - props_ok = prop_f(LEIA_NEURD_PROP_CONVERGENCE, clampf(dp.convergence, -0.2f, 0.2f), g.p_conv); + props_ok = prop_f(NEURD_PROP_CONVERGENCE, clampf(dp.convergence, -0.2f, 0.2f), g.p_conv); } g.props_valid = props_ok; if (!props_ok) { @@ -1491,33 +1502,33 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, } // ---- 5. Convert (blocking). - struct leia_neurd_image nin = {}; + struct NeurD_image nin = {}; nin.data = s->in_buf; nin.width = (int32_t)w; nin.height = (int32_t)h; nin.stride = (int32_t)(w * 4); - nin.pix_fmt = LEIA_NEURD_PIXEL_FORMAT_RGBA8; - struct leia_neurd_image nout = {}; - enum leia_neurd_status st; + nin.pix_fmt = NEURD_PIXEL_FORMAT_RGBA8; + struct NeurD_image nout = {}; + enum NeurD_status st; if (!vp.empty() && !g.interactive_unavailable) { - st = nd->convert_stream_dx_interactive(s->ns, &nin, vp.data(), (int)vp.size(), - LEIA_NEURD_PIXEL_FORMAT_RGBA8, &nout); - if (st == LEIA_NEURD_UNAVAILABLE_OUTDATED_RUNTIME) { + st = NeurD_convert_stream_dx_interactive(nd, s->ns, &nin, vp.data(), (int)vp.size(), + NEURD_PIXEL_FORMAT_RGBA8, &nout); + if (st == NEURD_UNAVAILABLE_OUTDATED_RUNTIME) { g.interactive_unavailable = true; U_LOG_W("Leia lift: NeurD interactive convert unavailable — tracked viewpoints ignored"); } - if (st != LEIA_NEURD_SUCCESS && s->mode == LEIA_LIFT_MODE_SBS) { + if (st != NEURD_SUCCESS && s->mode == LEIA_LIFT_MODE_SBS) { LIFT_WARN_ONCE("Leia lift: interactive convert -> %s; retrying with the default pattern", status_str(st)); - st = nd->convert_stream_dx(s->ns, &nin, LEIA_NEURD_PIXEL_FORMAT_RGBA8, &nout); + st = NeurD_convert_stream_dx(nd, s->ns, &nin, NEURD_PIXEL_FORMAT_RGBA8, &nout); } } else if (s->mode == LEIA_LIFT_MODE_NVIEW && g.interactive_unavailable) { LIFT_WARN_ONCE("Leia lift: N-view needs NeurD >= 0.4.5 (interactive convert)"); return false; } else { - st = nd->convert_stream_dx(s->ns, &nin, LEIA_NEURD_PIXEL_FORMAT_RGBA8, &nout); + st = NeurD_convert_stream_dx(nd, s->ns, &nin, NEURD_PIXEL_FORMAT_RGBA8, &nout); } - if (st != LEIA_NEURD_SUCCESS || nout.data == nullptr) { + if (st != NEURD_SUCCESS || nout.data == nullptr) { LIFT_WARN_ONCE("Leia lift: NeurD convert failed: %s", status_str(st)); return false; } diff --git a/src/drv_leia/leia_neurd_abi.h b/src/drv_leia/leia_neurd_abi.h deleted file mode 100644 index 02d4056..0000000 --- a/src/drv_leia/leia_neurd_abi.h +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright 2026, Leia Inc. -// SPDX-License-Identifier: Apache-2.0 -/*! - * @file - * @brief Minimal binary-interface declarations for the NeurD runtime DLL - * (Leia's 2D->3D conversion library), used by leia_lift_neurd.cpp. - * - * NeurD ships as a DLL whose only exported symbol is `NeurD_load`; it returns a - * pointer to a versioned table of function pointers. This header declares just - * enough of that contract to drive the D3D11 stream path — the enums and - * structs the plug-in passes by value, and the table layout. It is written - * against NeurD 0.4.6 and contains no NeurD code. - * - * ABI rules this relies on (NeurD's own compatibility policy): - * - table entries are append-only, never reordered or re-signatured; - * - `version` in the table is the LOADED runtime's version, so an entry may - * only be read when `version >= the entry's introduction version` (an older - * runtime's table is physically shorter — reading past it is UB). Use - * LEIA_NEURD_HAS() for every call. - * - * If NeurD ever breaks this (it would be a NeurD major), the version check in - * leia_lift_neurd.cpp rejects a major other than 0 and lift reports unavailable. - * - * @ingroup drv_leia - */ - -#pragma once - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(_WIN32) -#define LEIA_NEURD_CALL __stdcall -#else -#define LEIA_NEURD_CALL -#endif - -#define LEIA_NEURD_MAKE_VERSION(major, minor, patch) \ - ((((uint64_t)(major)&0xffffULL) << 48) | (((uint64_t)(minor)&0xffffULL) << 32) | \ - ((uint64_t)(patch)&0xffffffffULL)) -#define LEIA_NEURD_VERSION_MAJOR(v) ((uint32_t)(((uint64_t)(v) >> 48) & 0xffffULL)) -#define LEIA_NEURD_VERSION_MINOR(v) ((uint32_t)(((uint64_t)(v) >> 32) & 0xffffULL)) -#define LEIA_NEURD_VERSION_PATCH(v) ((uint32_t)((uint64_t)(v)&0xffffffffULL)) - -//! The table revision this header describes (requested in NeurD_load). -#define LEIA_NEURD_BUILT_AGAINST LEIA_NEURD_MAKE_VERSION(0, 4, 6) - -enum leia_neurd_status -{ - LEIA_NEURD_SUCCESS = 0, - LEIA_NEURD_IN_PROGRESS = 1, - LEIA_NEURD_GENERIC_ERROR = -1, - LEIA_NEURD_INVALID_ARG = -2, - LEIA_NEURD_INVALID_LICENSE = -3, - LEIA_NEURD_UNAVAILABLE_OUTDATED_RUNTIME = -4, - LEIA_NEURD_NOT_INITIALIZED = -5, - LEIA_NEURD_LICENSE_NETWORK_ERROR = -6, - LEIA_NEURD_INVALID_MODEL = -7, - LEIA_NEURD_FILESYSTEM_ERROR = -8, - LEIA_NEURD_NETWORK_ERROR = -9, - LEIA_NEURD_ABORTED = -10, - LEIA_NEURD_STATUS_MAKE_32BIT = 0x7FFFFFFF -}; - -enum leia_neurd_pixel_format -{ - LEIA_NEURD_PIXEL_FORMAT_NV12 = 0, - LEIA_NEURD_PIXEL_FORMAT_BGR8 = 1, - LEIA_NEURD_PIXEL_FORMAT_BGR32 = 2, - LEIA_NEURD_PIXEL_FORMAT_RGBA8 = 3, - LEIA_NEURD_PIXEL_FORMAT_RGB8 = 4, - LEIA_NEURD_PIXEL_FORMAT_MAKE_32BIT = 0x7FFFFFFF -}; - -//! For the DX entry points `data` is an ID3D11Buffer* (input) and an -//! ID3D11Buffer* or ID3D11Texture2D* (output, owned by NeurD). -struct leia_neurd_image -{ - void *data; - int32_t width; - int32_t height; - int32_t stride; - enum leia_neurd_pixel_format pix_fmt; -}; - -enum leia_neurd_output_type -{ - LEIA_NEURD_OUTPUT_SBS = 0, - LEIA_NEURD_OUTPUT_TB = 1, - LEIA_NEURD_OUTPUT_DEPTH = 2, -}; - -enum leia_neurd_autoscaling -{ - LEIA_NEURD_AUTOSCALING_NONE = 0, - LEIA_NEURD_AUTOSCALING_720P = 1, - LEIA_NEURD_AUTOSCALING_1080P = 2, - LEIA_NEURD_AUTOSCALING_1440P = 3, -}; - -enum leia_neurd_inpaint -{ - LEIA_NEURD_INPAINT_V1_STRETCH = 0, - LEIA_NEURD_INPAINT_V1_BLUR = 1, -}; - -enum leia_neurd_model_id -{ - LEIA_NEURD_MODEL_PHOTO_RELATIVE_QUALITY = 0, - LEIA_NEURD_MODEL_VIDEO_RELATIVE_FAST = 1, - LEIA_NEURD_MODEL_VIDEO_METRIC_QUALITY = 2, -}; - -struct leia_neurd_init_options -{ - uint32_t struct_size; - int32_t photo_model_id; - int32_t video_model_id; -}; - -//! Property ids (NeurD_prop). All are process-global inside NeurD. -enum leia_neurd_prop -{ - LEIA_NEURD_PROP_OUTPUT_TYPE = 0, //!< int, leia_neurd_output_type - LEIA_NEURD_PROP_GAIN_MULTIPLIER = 2, //!< float, default 1 - LEIA_NEURD_PROP_CONVERGENCE = 3, //!< float, must be in [-0.2, 0.2] - LEIA_NEURD_PROP_INPUT_AUTOSCALING = 7, //!< int, leia_neurd_autoscaling (default 1440p) - LEIA_NEURD_PROP_INPAINT_TYPE = 8, //!< int, leia_neurd_inpaint - LEIA_NEURD_PROP_OUTPUT_TILES_W = 9, //!< int, default 1 - LEIA_NEURD_PROP_OUTPUT_TILES_H = 10, //!< int, default 1 - LEIA_NEURD_PROP_AUTO_CONVERGENCE = 11, //!< int bool, default true - LEIA_NEURD_PROP_MAKE_32BIT = 0x7FFFFFFF -}; - -enum leia_neurd_backend -{ - LEIA_NEURD_BACKEND_CUDA = 0, - LEIA_NEURD_BACKEND_DIRECTML = 1, - LEIA_NEURD_BACKEND_OPENVINO = 2, - LEIA_NEURD_BACKEND_MAKE_32BIT = 0x7FFFFFFF -}; - -struct leia_neurd_load_request -{ - uint64_t version; -}; - -struct leia_neurd_stream; // opaque - -typedef void *leia_neurd_fn_unused; - -/*! - * The NeurD function table, in NeurD's table order (append-only). Entries the - * plug-in never calls are typed as leia_neurd_fn_unused — they only hold their - * slot. The trailing comment is the version that introduced each entry. - */ -struct leia_neurd_table -{ - uint64_t version; - - enum leia_neurd_status(LEIA_NEURD_CALL *set_prop_1f)(enum leia_neurd_prop prop, float value); // 0.2.1 - enum leia_neurd_status(LEIA_NEURD_CALL *set_prop_1i)(enum leia_neurd_prop prop, int32_t value); // 0.2.1 - leia_neurd_fn_unused deprecated_init_license; // 0.2.1 - void(LEIA_NEURD_CALL *deinit)(void); // 0.2.1 - leia_neurd_fn_unused convert_cuda; // 0.2.1 - enum leia_neurd_status(LEIA_NEURD_CALL *init)(int32_t multithreaded); // 0.2.2 - leia_neurd_fn_unused get_prop_1f; // 0.2.2 - leia_neurd_fn_unused get_prop_1i; // 0.2.2 - enum leia_neurd_status(LEIA_NEURD_CALL *set_logger_callback)(void (*cb)(const char *msg, int size)); // 0.2.5 - leia_neurd_fn_unused get_cuda_runtime_version; // 0.2.7 - leia_neurd_fn_unused read_multiview; // 0.3.2 - leia_neurd_fn_unused get_multiview_attributes; // 0.3.2 - leia_neurd_fn_unused render_multiview; // 0.3.2 - leia_neurd_fn_unused render_multiview_interactive_cpu; // 0.3.2 - leia_neurd_fn_unused render_multiview_interactive_cuda; // 0.3.2 - leia_neurd_fn_unused free_multiview; // 0.3.2 - enum leia_neurd_status(LEIA_NEURD_CALL *set_backend)(enum leia_neurd_backend backend); // 0.3.4 - enum leia_neurd_status(LEIA_NEURD_CALL *get_backend)(enum leia_neurd_backend *backend); // 0.3.4 - leia_neurd_fn_unused convert_cpu; // 0.3.5 - leia_neurd_fn_unused prepare_multiview; // 0.3.7 - leia_neurd_fn_unused convert_dx; // 0.3.8 - void(LEIA_NEURD_CALL *shrink_memory_pool)(void); // 0.3.10 - enum leia_neurd_status(LEIA_NEURD_CALL *create_stream)(struct leia_neurd_stream **stream); // 0.3.11 - enum leia_neurd_status(LEIA_NEURD_CALL *destroy_stream)(struct leia_neurd_stream *stream); // 0.3.11 - leia_neurd_fn_unused convert_stream_cpu; // 0.3.11 - leia_neurd_fn_unused convert_stream_cuda; // 0.3.11 - enum leia_neurd_status(LEIA_NEURD_CALL *convert_stream_dx)(struct leia_neurd_stream *stream, - const struct leia_neurd_image *input, - enum leia_neurd_pixel_format output_pix_fmt, - struct leia_neurd_image *output); // 0.3.11 - leia_neurd_fn_unused convert_batch_cpu; // 0.4.3 - leia_neurd_fn_unused convert_batch_cuda; // 0.4.3 - leia_neurd_fn_unused convert_batch_dx; // 0.4.3 - void *(LEIA_NEURD_CALL *get_dx_device)(void); // 0.4.3 - leia_neurd_fn_unused convert_stream_cuda_interactive; // 0.4.5 - enum leia_neurd_status(LEIA_NEURD_CALL *convert_stream_dx_interactive)( - struct leia_neurd_stream *stream, - const struct leia_neurd_image *input, - const float *viewpoint_xyz, - int size, - enum leia_neurd_pixel_format output_pix_fmt, - struct leia_neurd_image *output); // 0.4.5 - leia_neurd_fn_unused convert_batch_cuda_interactive; // 0.4.5 - leia_neurd_fn_unused convert_batch_dx_interactive; // 0.4.5 - enum leia_neurd_status(LEIA_NEURD_CALL *init_with_options)(int32_t multithreaded, - const struct leia_neurd_init_options *opts); // 0.4.6 -}; - -typedef const struct leia_neurd_table *(LEIA_NEURD_CALL *leia_neurd_load_fn)(const struct leia_neurd_load_request *req); - -/*! - * True when the loaded table carries @p field. The version test comes FIRST and - * short-circuits, so a shorter (older) table is never read past its end. - */ -#define LEIA_NEURD_HAS(tbl, field, maj, min, pat) \ - ((tbl) != NULL && (tbl)->version >= LEIA_NEURD_MAKE_VERSION(maj, min, pat) && (tbl)->field != NULL) - -#ifdef __cplusplus -} -#endif From 3f4dfc57f783ba85d4fd91216e8e24f76634e1c3 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 13:58:47 -0700 Subject: [PATCH 08/21] feat(lift): lift-only D3D11 DP factory + resync to the final lift header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - create_dp_d3d11_lift (xrt_plugin.h, ADR-042): a DP with NO SR weaver, window, tracker or lens control — only the NeurD lift slots; every other slot NULL. Filled when the iface header has XRT_PLUGIN_IFACE_HAS_D3D11_LIFT_FACTORY and the NeurD headers were fetched; otherwise NULL (runtime falls back). - convergence is now the runtime's relative depth in [0,1] (<0 = auto), mapped to NeurD's [-0.2,0.2] as K*(c-0.5), K = DXR_LEIA_LIFT_CONV_GAIN (default 0.4; uncalibrated, calibration recipe in docs/lift-neurd.md). - focal_px (appended) ignored; lift_convert_blob stays NULL. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/lift-neurd.md | 46 ++++++++++------ src/drv_leia/leia_display_processor_d3d11.cpp | 52 +++++++++++++++++++ src/drv_leia/leia_display_processor_d3d11.h | 13 +++++ src/drv_leia/leia_lift_neurd.cpp | 21 +++++++- src/drv_leia/leia_lift_neurd.h | 3 +- src/drv_leia/leia_plugin.c | 15 ++++++ 6 files changed, 131 insertions(+), 19 deletions(-) diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index b427689..132d85b 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -26,6 +26,19 @@ carries bit 8. | `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device — `R8G8B8A8_UNORM` for SBS/NVIEW, `R8_UNORM` for DEPTH — owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | Every versioned struct is read only as far as its `struct_size` covers. +`xrt_dp_lift_params.focal_px` (appended) is ignored — it only matters to a +photo → Gaussians module, and `lift_convert_blob` stays NULL. + +### The lift-only DP (`create_dp_d3d11_lift`) + +The plug-in fills the optional `xrt_plugin_iface::create_dp_d3d11_lift` factory +(runtime header macro `XRT_PLUGIN_IFACE_HAS_D3D11_LIFT_FACTORY`). The runtime creates +exactly one lift DP per process on a dedicated device, calls only the `lift_*` slots on +it from its lift thread, and always passes explicit `viewpoints_xyz`. So this DP has +**no SR weaver, no window, no tracker, no lens control** — just the NeurD handle; every +non-lift slot is NULL. This is what keeps a lift session from perturbing the lens. +The weaving DP (`create_dp_d3d11`) still carries the lift slots too (they load nothing +unless called), which is what a runtime without the lift factory falls back to. Output layouts (runtime contract): DEPTH = one channel at the inference resolution (NeurD writes the normalised disparity replicated into RGB; the unpack keeps R); @@ -137,7 +150,7 @@ how its `z` relates to viewing distance, so mapping it would be a guess. Viewpoint source, in order: -1. **Explicit viewpoints** from the runtime (recommended — see *Known limits*) (`viewpoints_xyz`, metres, display space): +1. **Explicit viewpoints** from the runtime (always sent to the lift-only DP) (`viewpoints_xyz`, metres, display space): `view_count` triplets are used 1:1; exactly two are treated as an eye pair. 2. **Tracked eyes** from the SR eye path, when tracking is live (inter-eye distance > 1 mm — the same tracking-loss test as the eye slot). @@ -153,15 +166,20 @@ default pattern and NVIEW is refused. | `xrt_dp_lift_params` | NeurD | |---|---| | `convergence < 0` | `AUTO_CONVERGENCE = TRUE` | -| `convergence ≥ 0` | `AUTO_CONVERGENCE = FALSE`, `CONVERGENCE` clamped to NeurD's [−0.2, 0.2] | +| `convergence` in [0, 1] | `AUTO_CONVERGENCE = FALSE`, `CONVERGENCE = clamp(K · (c − 0.5), ±0.2)`, K = `DXR_LEIA_LIFT_CONV_GAIN` | | `strength` | `GAIN_MULTIPLIER` (1 = NeurD's calibrated budget, 0 = flat), clamped [0, 10]; negative → 1.0 | | `inpaint` | NeurD always fills disocclusions: 0 → `V1_STRETCH` (cheapest), non-zero → `V1_BLUR` (the DX video path supports V1 only) | | `view_count` | NVIEW row width (2..8) | -**Convergence units are not reconciled yet.** The runtime defines `convergence` as -"relative depth placed at the display plane"; NeurD's `CONVERGENCE` is its own -disparity offset in [−0.2, 0.2]. The value is passed through clamped; the mapping needs -calibrating on a panel (auto-convergence, the default, is unaffected). +**Convergence calibration.** The runtime's `convergence` is the relative depth placed +at the display plane, normalised to [0, 1] over the frame's depth range (0 = nearest +content on the glass, 1 = farthest, 0.5 = middle). NeurD's `CONVERGENCE` is its own +disparity offset in [−0.2, 0.2]. The plug-in maps linearly about mid-range with gain +K (default 0.4, which spans NeurD's whole range). This is **uncalibrated**: on a panel, +submit c = 0 and check that the nearest content sits on the glass; if the far content +does instead, the sign is reversed — set `DXR_LEIA_LIFT_CONV_GAIN=-0.4`; then tune |K| +until c = 0 and c = 1 put the extremes on the glass. Auto-convergence (the default, +c < 0) is unaffected. `xrt_dp_lift_stream_info.input_scale` in (0, 1] (1 = native) picks the smallest NeurD autoscaling bucket (720p / 1080p / 1440p / none) that covers `input_scale × input @@ -183,6 +201,7 @@ Read once per DP at create (`leia_lift_neurd_create`). | `DXR_LEIA_LIFT_BACKEND` | `directml` | `auto` \| `directml` \| `cuda` \| `openvino`. First activation in the process wins (NeurD's forced backend is sticky). Only DirectML yields a D3D11 device. | | `DXR_LEIA_LIFT_SCALE` | unset (→ stream `input_scale`, else 720p) | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. When set it overrides every stream's `input_scale`. NeurD's own default is 1440p; 720p is the fallback for latency. | | `DXR_LEIA_LIFT_VIEW_GAIN` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | +| `DXR_LEIA_LIFT_CONV_GAIN` | `0.4` | `K` in the convergence map above, [−2, 2]; negative flips the sign. Calibration knob. | Under the service, remember these are read by `displayxr-service.exe`'s environment, not the client's. @@ -263,17 +282,10 @@ lift-enabled runtime + this plug-in registered: ## Known limits -- **The runtime's dedicated lift DP gets a full SR weaver.** The runtime calls the - lift slots on a DP it creates *for* lift, with a NULL window, and asks that such a DP - be cheap. This plug-in cannot tell that DP from a NULL-HWND `_texture` session DP - (which must weave), so it creates the usual async SR weaver for it. Two - consequences: (1) a second weaver on the shared SR context may briefly perturb the - lens (the effect the #625 snap probe guards against), and (2) the tracked eyes the - lift reads come from that extra weaver. Recommended: the runtime passes explicit - `viewpoints_xyz` (the panel DP's predicted eyes) on every convert, and a follow-up - gives the DP a lift-only signal (or makes NULL-window weaver creation lazy, on the - first `process_atlas`) so the lift DP never builds a weaver. - +- **Fallback path only:** against a runtime without `create_dp_d3d11_lift`, the runtime + calls the ordinary factory with a NULL window, which builds an async SR weaver (it + cannot be told apart from a NULL-HWND `_texture` DP). That extra weaver can briefly + perturb the lens. The lift-only factory removes this on runtimes that have it. - Windows / D3D11 only. The D3D12, GL and Vulkan DPs do not implement lift. - The activation thread is detached; unloading the plug-in DLL while it runs (a first activation in progress) is unsafe. The runtime does not unload plug-ins diff --git a/src/drv_leia/leia_display_processor_d3d11.cpp b/src/drv_leia/leia_display_processor_d3d11.cpp index f0191df..7ecd1d1 100644 --- a/src/drv_leia/leia_display_processor_d3d11.cpp +++ b/src/drv_leia/leia_display_processor_d3d11.cpp @@ -2743,6 +2743,58 @@ leia_dp_factory_d3d11(void *d3d11_device, } +#ifdef DXR_LEIA_DP_D3D11_LIFT +/* + * + * Lift-only factory — xrt_plugin_iface::create_dp_d3d11_lift (ADR-042). + * + * The runtime creates exactly one of these per process on a dedicated device + * and drives ONLY the lift_* slots on it, from its lift thread, always with + * explicit viewpoints. So: no SR weaver (a second weaver on the shared SR + * context perturbs the lens), no window, no tracker, no lens control, no blit + * shaders — just the NeurD lift handle. Every non-lift slot stays NULL, which + * the runtime's slot helpers read as "absent". NeurD itself is still loaded + * lazily, on the first lift call. + * + */ + +extern "C" xrt_result_t +leia_dp_factory_d3d11_lift(void *d3d11_device, + void *d3d11_context, + void *window_handle, + struct xrt_display_processor_d3d11 **out_xdp) +{ + (void)d3d11_context; + (void)window_handle; // always NULL for the lift DP + if (out_xdp == NULL) { + return XRT_ERROR_DEVICE_CREATION_FAILED; + } + struct leia_display_processor_d3d11_impl *ldp = + (struct leia_display_processor_d3d11_impl *)calloc(1, sizeof(*ldp)); + if (ldp == NULL) { + return XRT_ERROR_ALLOCATION; + } + ldp->base.struct_size = static_cast(sizeof(struct xrt_display_processor_d3d11)); + ldp->base.destroy = leia_dp_d3d11_destroy; // NULL-safe for every member left unset here + ldp->base.lift_get_caps = leia_dp_d3d11_lift_get_caps; + ldp->base.lift_stream_create = leia_dp_d3d11_lift_stream_create; + ldp->base.lift_stream_destroy = leia_dp_d3d11_lift_stream_destroy; + ldp->base.lift_convert = leia_dp_d3d11_lift_convert; + // lift_convert_blob (GAUSSIANS) stays NULL: NeurD has no splat path. + ldp->device = static_cast(d3d11_device); + ldp->lift = leia_lift_neurd_create(); // reads DXR_LEIA_LIFT* once; loads nothing + ldp->view_count = 2; + ldp->async_zone_publish = true; + // leiasr stays NULL: lift_convert then uses only the runtime's explicit + // viewpoints (or NeurD's default pattern) — never a tracker. + + *out_xdp = &ldp->base; + U_LOG_W("Created Leia D3D11 LIFT-ONLY display processor (no weaver, no window; NeurD loaded on first use)"); + return XRT_SUCCESS; +} +#endif // DXR_LEIA_DP_D3D11_LIFT + + /* * * Legacy creation function — wraps an existing leiasr_d3d11 handle. diff --git a/src/drv_leia/leia_display_processor_d3d11.h b/src/drv_leia/leia_display_processor_d3d11.h index 7a74691..41c1927 100644 --- a/src/drv_leia/leia_display_processor_d3d11.h +++ b/src/drv_leia/leia_display_processor_d3d11.h @@ -35,6 +35,19 @@ leia_dp_factory_d3d11(void *d3d11_device, void *window_handle, struct xrt_display_processor_d3d11 **out_xdp); +#if defined(DXR_LEIA_HAS_NEURD) +/*! + * Lift-only D3D11 DP (xrt_plugin_iface::create_dp_d3d11_lift, ADR-042): no + * weaver, no window, only the NeurD lift_* slots. Defined only when the + * runtime headers carry the lift slots AND the NeurD headers were fetched. + */ +xrt_result_t +leia_dp_factory_d3d11_lift(void *d3d11_device, + void *d3d11_context, + void *window_handle, + struct xrt_display_processor_d3d11 **out_xdp); +#endif + /*! * Create an @ref xrt_display_processor_d3d11 that wraps a Leia SR SDK * D3D11 weaver (leiasr_d3d11_set_input_texture + leiasr_d3d11_weave). diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index eb19529..4303cc3 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -186,6 +186,7 @@ struct knobs int32_t autoscaling; //!< DXR_LEIA_LIFT_SCALE (default 720p) bool scale_forced; //!< DXR_LEIA_LIFT_SCALE was set: overrides the stream's input_scale float view_gain; //!< DXR_LEIA_LIFT_VIEW_GAIN (default 1.0) + float conv_gain; //!< DXR_LEIA_LIFT_CONV_GAIN (default 0.4): relative convergence -> NeurD units }; bool @@ -202,6 +203,7 @@ read_knobs() k.backend = BACKEND_DIRECTML; k.autoscaling = NEURD_INPUT_AUTOSCALING_720P; k.view_gain = 1.0f; + k.conv_gain = 0.4f; const char *e = std::getenv("DXR_LEIA_LIFT"); if (e != nullptr && (e[0] == '0' || env_ieq(e, "off") || env_ieq(e, "false"))) { @@ -250,6 +252,15 @@ read_knobs() U_LOG_W("Leia lift: DXR_LEIA_LIFT_VIEW_GAIN='%s' out of range [0,10] — using 1.0", e); } } + e = std::getenv("DXR_LEIA_LIFT_CONV_GAIN"); + if (e != nullptr && e[0] != '\0') { + float cg = (float)std::atof(e); + if (std::isfinite(cg) && cg >= -2.0f && cg <= 2.0f) { + k.conv_gain = cg; + } else { + U_LOG_W("Leia lift: DXR_LEIA_LIFT_CONV_GAIN='%s' out of range [-2,2] — using 0.4", e); + } + } return k; } @@ -1441,7 +1452,15 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, prop_i(NEURD_PROP_AUTO_CONVERGENCE, auto_conv ? 1 : 0, g.p_autoconv) && prop_f(NEURD_PROP_GAIN_MULTIPLIER, gain, g.p_gain); if (props_ok && !auto_conv) { - props_ok = prop_f(NEURD_PROP_CONVERGENCE, clampf(dp.convergence, -0.2f, 0.2f), g.p_conv); + // Runtime: convergence = RELATIVE depth placed at the display plane, + // [0,1] over the frame's depth range (0 = nearest on the glass, 1 = + // farthest). NeurD: a disparity offset in [-0.2, 0.2]. Linear map + // about the mid-range, K = DXR_LEIA_LIFT_CONV_GAIN (default 0.4 spans + // NeurD's full range; negate K if the sign proves reversed on a + // panel). UNCALIBRATED — see docs/lift-neurd.md. + const float c = clampf(dp.convergence, 0.0f, 1.0f); + const float nd_conv = clampf(l->k.conv_gain * (c - 0.5f), -0.2f, 0.2f); + props_ok = prop_f(NEURD_PROP_CONVERGENCE, nd_conv, g.p_conv); } g.props_valid = props_ok; if (!props_ok) { diff --git a/src/drv_leia/leia_lift_neurd.h b/src/drv_leia/leia_lift_neurd.h index 1b13a29..e5956d5 100644 --- a/src/drv_leia/leia_lift_neurd.h +++ b/src/drv_leia/leia_lift_neurd.h @@ -66,7 +66,8 @@ struct leia_lift_neurd_stream_desc struct leia_lift_neurd_params { - float convergence; //!< <0 = NeurD auto-convergence; else NeurD units, clamped to [-0.2, 0.2]. + float convergence; //!< <0 = NeurD auto-convergence; else relative depth at the display plane + //!< in [0,1], mapped to NeurD units by DXR_LEIA_LIFT_CONV_GAIN. float strength; //!< NeurD gain multiplier: 1 = calibrated budget, 0 = flat; <0 = 1.0; max 10. uint32_t inpaint; //!< 0 stretch fill, non-zero blur fill. uint32_t view_count; //!< NVIEW only (2..max_views). diff --git a/src/drv_leia/leia_plugin.c b/src/drv_leia/leia_plugin.c index bde7935..1ad54ca 100644 --- a/src/drv_leia/leia_plugin.c +++ b/src/drv_leia/leia_plugin.c @@ -437,6 +437,21 @@ static struct xrt_plugin_iface g_leia_iface = { .set_pose_source = leia_plugin_set_pose_source, .probe_displays = leia_plugin_probe_displays, + + /* + * ADR-042 lift-only D3D11 DP: no weaver / window / tracker, only the NeurD + * lift slots (docs/lift-neurd.md). Needs the runtime header's field AND the + * NeurD headers (DXR_LEIA_HAS_NEURD); otherwise the field is absent or NULL + * and the runtime has no lift from this plug-in. The iface header carrying + * this field implies its DP header carries XRT_DP_D3D11_HAS_LIFT. + */ +#if defined(XRT_PLUGIN_IFACE_HAS_D3D11_LIFT_FACTORY) +#if defined(XRT_HAVE_LEIA_SR_D3D11) && defined(DXR_LEIA_HAS_NEURD) + .create_dp_d3d11_lift = leia_dp_factory_d3d11_lift, +#else + .create_dp_d3d11_lift = NULL, +#endif +#endif }; From d4498685e4bf4845f031da7065695c99fc495dd0 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 19:30:59 -0700 Subject: [PATCH 09/21] =?UTF-8?q?fix(lift):=20log=20NeurD's=20activation/i?= =?UTF-8?q?nit=20callback=20lines=20at=20WARN=20until=20READY=20=E2=80=94?= =?UTF-8?q?=20the=20only=20place=20NeurD=20says=20why=20init=20failed=20wa?= =?UTF-8?q?s=20dropped=20at=20the=20default=20log=20level?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drv_leia/leia_lift_neurd.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index 4303cc3..5cf92e1 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -458,7 +458,14 @@ neurd_log_cb(const char *msg, int size) while (size > 0 && (msg[size - 1] == '\n' || msg[size - 1] == '\r')) { size--; } - U_LOG_I("NeurD: %.*s", size, msg); + // Until the module is READY these are one-off activation/init lines (licence, model paths, + // adapter) and the only place NeurD says WHY an init failed — surface them at the default + // log level. Once READY they can be per-stream chatter, so drop back to INFO. + if (g.state.load() != G_READY) { + U_LOG_W("NeurD: %.*s", size, msg); + } else { + U_LOG_I("NeurD: %.*s", size, msg); + } } bool From d911a98a3688e3c79c357f6325610c43df227fc2 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 20:24:57 -0700 Subject: [PATCH 10/21] =?UTF-8?q?feat(lift):=20run=20on=20NeurD=200.3.11+?= =?UTF-8?q?=20=E2=80=94=20get=5Fdx=5Fdevice=20optional,=20own=20bridge=20d?= =?UTF-8?q?evice=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NeurD 0.3.12 is the officially supported 2D->3D line but lacks NeurD_get_dx_device (introduced 0.4.3), which the lift module hard-gated on. ABI facts (media_sdk tags v0.3.12 / v0.4.6): - The function table (_NeurD_table.h) is append-only: 0.3.12's 27 entries are an exact prefix of 0.4.6's 36. NeurD_load in 0.3.12 does no version check and returns its table with version = 0.3.12, so a plug-in built against the 0.4.6 header loads it, and LEIA_NEURD_HAS (version test first) correctly reports the 0.4.x entries absent. - 0.3.12 fills create_stream, destroy_stream, convert_stream_dx, set_backend, get_backend, shrink_memory_pool, set_logger_callback, set_prop_1i/1f, init. Changes: - activation gate: drop get_dx_device; need 0.3.11+ (the stream API). - setup_nd_device_locked: use get_dx_device when present; otherwise create the plug-in's own D3D11 device on the default adapter (as NeurD's example does), flagged nd_dev_owned so the setup-failure path Releases it (release_nd_dev_locked). One WARN names the case + LUID. - caps: drop NVIEW once READY on a NeurD without interactive convert (< 0.4.5); interactive_unavailable is now atomic since caps reads it lock-free. Caveat: in 0.3.12, media_sdk_convert_dx is compiled only under USE_TRT (CUDA): it GetDevice()s the caller's input buffer, binds CUDA-D3D11 interop to that device's adapter, and returns an R8G8B8A8_UNORM texture on the same device — which the existing stage_output texture branch already copies into the bridge. The DirectML build's DX path is unverified, so on 0.3.x the service must run with DXR_LEIA_LIFT_BACKEND=cuda, and the default adapter must be the NVIDIA GPU. 0.3.x is SBS/DEPTH only, default models (NeurD_init). Documented in docs/lift-neurd.md. Not compiled or run on Windows yet. Co-Authored-By: Claude Fable 5.1 --- docs/lift-neurd.md | 37 +++++++++++--- src/drv_leia/leia_lift_neurd.cpp | 85 ++++++++++++++++++++++++++------ 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index 132d85b..fb1a235 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -20,7 +20,7 @@ carries bit 8. | Slot | Plug-in behaviour | |---|---| -| `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS\|NVIEW (1\|2\|4) once NeurD is present, else 0. `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 8, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | +| `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS\|NVIEW (1\|2\|4) once NeurD is present, else 0; NVIEW is dropped once READY on a NeurD without interactive convert (< 0.4.5). `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 8, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | | `lift_stream_create` | Non-blocking. Succeeds while NeurD is still activating — the NeurD stream is created lazily on the first convert. | | `lift_stream_destroy` | Releases the stream's NeurD stream and bridge resources. | | `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device — `R8G8B8A8_UNORM` for SBS/NVIEW, `R8_UNORM` for DEPTH — owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | @@ -82,10 +82,35 @@ Each transition logs one WARN (`Leia lift: ...`). ### Backend `NeurD_set_backend` is called immediately before init; NeurD keeps a forced backend for -the process, so the **first** DP to activate NeurD decides it. The D3D11 path needs -NeurD's D3D11 device (`NeurD_get_dx_device`), which only the **DirectML** backend -provides — CUDA and OpenVINO return none, and lift then reports unavailable with a WARN -naming the fix. Hence the default is `directml`, not NeurD's own `auto`. +the process, so the **first** DP to activate NeurD decides it. On NeurD 0.4.3+ the D3D11 +path needs NeurD's D3D11 device (`NeurD_get_dx_device`), which only the **DirectML** +backend provides — CUDA and OpenVINO return none, and lift then reports unavailable with +a WARN naming the fix. Hence the default is `directml`, not NeurD's own `auto`. NeurD +0.3.x is the other way round — see below. + +### NeurD 0.3.x + +Supported from **0.3.11** (the first release with the stream API: `create_stream`, +`convert_stream_dx`, `set_prop_1i/1f`). The function table is append-only, so a plug-in +built against the 0.4.x header loads a 0.3.x `NeurD.dll` unchanged, and every newer entry +is version-gated (`LEIA_NEURD_HAS`). + +- **`NeurD_get_dx_device` is optional** (added in 0.4.3). Without it the plug-in creates + its own D3D11 device on the **default adapter** (as NeurD's own example does), builds + the bridge on it, and releases it itself. 0.3.x's DX convert takes the device from the + input buffer (`GetDevice`) and returns its output as an `R8G8B8A8_UNORM` texture on that + same device, so the bridge is otherwise unchanged. One WARN names the case and the LUID. +- **CUDA backend required for the DX path on 0.3.x.** Its DX convert is implemented on + CUDA↔D3D11 interop; the DirectML build's DX path is not verified. Set + `DXR_LEIA_LIFT_BACKEND=cuda` in the **service's** environment (`displayxr-service.exe` + reads its own env, not the client's). The default adapter must be the NVIDIA GPU for + the interop to bind — on a hybrid box, pin the service to the dGPU. +- **SBS and DEPTH only.** N-view needs interactive convert (0.4.5+); caps drop NVIEW once + ready, and tracked-eye viewpoints are ignored (SBS uses NeurD's default pattern). + Because 0.3.x hands back a texture, DEPTH arrives as `R8G8B8A8_UNORM` (copied as-is), + not `R8_UNORM`; `lift_convert`'s `out_format` reports which. +- **Default models only** — `init_with_options` (model selection) is newer; `NeurD_init` + is used instead. ## Threading @@ -198,7 +223,7 @@ Read once per DP at create (`leia_lift_neurd_create`). | Env | Default | Effect | |---|---|---| | `DXR_LEIA_LIFT` | on | `0` / `off` → caps `modes=0, state=0`; NeurD is never probed or loaded. | -| `DXR_LEIA_LIFT_BACKEND` | `directml` | `auto` \| `directml` \| `cuda` \| `openvino`. First activation in the process wins (NeurD's forced backend is sticky). Only DirectML yields a D3D11 device. | +| `DXR_LEIA_LIFT_BACKEND` | `directml` | `auto` \| `directml` \| `cuda` \| `openvino`. First activation in the process wins (NeurD's forced backend is sticky). On NeurD 0.4.3+ only DirectML yields a D3D11 device; on 0.3.x use `cuda` (see *NeurD 0.3.x*). | | `DXR_LEIA_LIFT_SCALE` | unset (→ stream `input_scale`, else 720p) | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. When set it overrides every stream's `input_scale`. NeurD's own default is 1440p; 720p is the fallback for latency. | | `DXR_LEIA_LIFT_VIEW_GAIN` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | | `DXR_LEIA_LIFT_CONV_GAIN` | `0.4` | `K` in the convergence map above, [−2, 2]; negative flips the sign. Calibration knob. | diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index 5cf92e1..e626364 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -20,7 +20,7 @@ * decides when to shrink NeurD's memory pool. * * - Device bridge (the LeiaMeet DxStereoConverter pattern). NeurD runs on ITS - * OWN D3D11 device (NeurD_get_dx_device), possibly on a different adapter + * OWN D3D11 device (NeurD_get_dx_device, 0.4.3+), possibly on a different adapter * than the runtime's service device on hybrid boxes. NeurD's DX input must * be a raw RGBA8 ID3D11Buffer with D3D11_RESOURCE_MISC_SHARED, and NeurD * hands its output back on the input buffer's device. Buffers do not share @@ -38,6 +38,12 @@ * Every hop is CPU-drained with an event query: NeurD's DirectML work runs * on its own queue, unordered against any D3D11 context. * + * NeurD 0.3.11-0.4.2 has the stream API but no get_dx_device. There the + * plug-in creates the "NeurD device" itself (default adapter, like the SDK's + * own example) and owns it; NeurD 0.3.x's CUDA DX path runs its D3D11-CUDA + * interop on whatever device the input buffer lives on and hands back an + * RGBA8 texture on that same device, so the bridge is unchanged. + * * - NeurD properties are process-global, so the (set props -> convert) pair * must be atomic across streams: one process-wide mutex serialises every * NeurD call and every use of NeurD's immediate context. @@ -347,7 +353,8 @@ struct global enum NeurD_backend backend = NEURD_BACKEND_DIRECTML; char backend_name[32] = {}; - ID3D11Device *nd_dev = nullptr; //!< NeurD-owned: never Released. + ID3D11Device *nd_dev = nullptr; //!< NeurD-owned (never Released) unless nd_dev_owned. + bool nd_dev_owned = false; //!< nd_dev is the plug-in's own device (NeurD < 0.4.3): Release it. ID3D11DeviceContext *nd_ctx = nullptr; ID3D11Query *nd_done = nullptr; ID3D11ComputeShader *cs_pack = nullptr; @@ -358,7 +365,8 @@ struct global int refcount = 0; uint64_t next_stream_id = 1; - bool interactive_unavailable = false; + //! Written under g.mtx; atomic because caps reads it lock-free. + std::atomic interactive_unavailable{false}; //! Last-applied global NeurD properties (avoid a worker round-trip per prop per frame). bool props_valid = false; @@ -528,16 +536,59 @@ compile_cs(ID3D11Device *dev, const char *src, const char *name) return cs; } +//! Drop g.nd_dev: Released only when the plug-in created it. Called with g.mtx held. +void +release_nd_dev_locked() +{ + if (g.nd_dev_owned) { + safe_release(g.nd_dev); + } + g.nd_dev = nullptr; + g.nd_dev_owned = false; +} + //! Build the NeurD-device side of the bridge. Called with g.mtx held. bool setup_nd_device_locked() { - g.nd_dev = static_cast(NeurD_get_dx_device(g.nd)); - if (g.nd_dev == nullptr) { - U_LOG_W("Leia lift: NeurD backend '%s' exposes no D3D11 device — the D3D11 lift path needs the " - "DirectML backend (DXR_LEIA_LIFT_BACKEND=directml). Lift unavailable.", - g.backend_name); - return false; + if (LEIA_NEURD_HAS(g.nd, get_dx_device)) { + g.nd_dev = static_cast(NeurD_get_dx_device(g.nd)); + g.nd_dev_owned = false; + if (g.nd_dev == nullptr) { + U_LOG_W("Leia lift: NeurD backend '%s' exposes no D3D11 device — the D3D11 lift path needs the " + "DirectML backend (DXR_LEIA_LIFT_BACKEND=directml). Lift unavailable.", + g.backend_name); + return false; + } + } else { + // NeurD < 0.4.3: no get_dx_device. NeurD takes the device from the + // input buffer (GetDevice) and returns its output on that device, so + // the plug-in supplies the device — default adapter, as NeurD's own + // example does. + ID3D11Device *dev = nullptr; + ID3D11DeviceContext *ctx = nullptr; + HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, D3D11_CREATE_DEVICE_BGRA_SUPPORT, + nullptr, 0, D3D11_SDK_VERSION, &dev, nullptr, &ctx); + if (FAILED(hr) || dev == nullptr || ctx == nullptr) { + U_LOG_W("Leia lift: NeurD < 0.4.3 and D3D11CreateDevice for the plug-in's bridge device failed " + "(0x%08x) — lift unavailable", + (unsigned)hr); + safe_release(ctx); + safe_release(dev); + return false; + } + // GetImmediateContext below takes its own reference. + ctx->Release(); + g.nd_dev = dev; + g.nd_dev_owned = true; + LUID l = {}; + const bool have_luid = device_luid(dev, &l); + U_LOG_W("Leia lift: NeurD < 0.4.3: no get_dx_device — using the plug-in's own D3D11 device on the " + "default adapter (LUID %08lx:%08lx)%s", + have_luid ? (unsigned long)l.HighPart : 0UL, have_luid ? (unsigned long)l.LowPart : 0UL, + g.backend == NEURD_BACKEND_CUDA ? "" + : " — NeurD 0.3.x's DX path needs the CUDA backend " + "(DXR_LEIA_LIFT_BACKEND=cuda); converts may fail"); } g.nd_dev->GetImmediateContext(&g.nd_ctx); enable_mt_protection(g.nd_ctx, "NeurD"); @@ -560,7 +611,7 @@ setup_nd_device_locked() safe_release(g.nd_done); safe_release(g.cb); safe_release(g.nd_ctx); - g.nd_dev = nullptr; + release_nd_dev_locked(); return false; } if (device_luid(g.nd_dev, &g.nd_luid)) { @@ -609,10 +660,9 @@ activation_worker() NEURD_GET_VERSION_MINOR(NEURD_VERSION), NEURD_GET_VERSION_PATCH(NEURD_VERSION)); if (NEURD_GET_VERSION_MAJOR(nd->version) != 0 || - !LEIA_NEURD_HAS(nd, convert_stream_dx) || !LEIA_NEURD_HAS(nd, get_dx_device) || - !LEIA_NEURD_HAS(nd, create_stream) || !LEIA_NEURD_HAS(nd, set_prop_1i) || - !LEIA_NEURD_HAS(nd, set_prop_1f)) { - U_LOG_W("Leia lift: NeurD runtime too old/new for the D3D11 stream path (need 0.4.3+, " + !LEIA_NEURD_HAS(nd, convert_stream_dx) || !LEIA_NEURD_HAS(nd, create_stream) || + !LEIA_NEURD_HAS(nd, set_prop_1i) || !LEIA_NEURD_HAS(nd, set_prop_1f)) { + U_LOG_W("Leia lift: NeurD runtime too old/new for the D3D11 stream path (need 0.3.11+, " "major 0) — lift unavailable"); g.state.store(G_FAILED); g.worker_running.store(false); @@ -1196,7 +1246,12 @@ leia_lift_neurd_get_caps(struct leia_lift_neurd *l, struct leia_lift_neurd_caps if (out->state == LEIA_LIFT_STATE_UNAVAILABLE) { return true; } - out->modes = LEIA_LIFT_MODE_DEPTH | LEIA_LIFT_MODE_SBS | LEIA_LIFT_MODE_NVIEW; + out->modes = LEIA_LIFT_MODE_DEPTH | LEIA_LIFT_MODE_SBS; + // N-view needs interactive convert (NeurD 0.4.5+). Known only once READY; + // before that, advertise it optimistically (it may still turn out absent). + if (s != G_READY || !g.interactive_unavailable.load()) { + out->modes |= LEIA_LIFT_MODE_NVIEW; + } out->max_streams = kMaxStreams; out->max_views = kMaxViews; out->depth_semantics = 0; // relative: per-frame min-max normalised disparity From 9b6114f2ff0a9c3d779449ef5f7d3509799eba3a Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 20:34:51 -0700 Subject: [PATCH 11/21] =?UTF-8?q?feat(lift):=20DXR=5FLEIA=5FLIFT=5FINTERAC?= =?UTF-8?q?TIVE=5FMIN=20=E2=80=94=20opt-in=20interactive=20gate=20for=20th?= =?UTF-8?q?e=200.4.4=20internal-interactive=20NeurD=20package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NeurD-Runtime-0.4.4-internal-interactive package reports version 0.4.4 but carries convert_stream_dx_interactive. In media_sdk the interactive API landed in 5b4faa8 (NEURDW-80) as a 35-entry table that is an exact prefix of 0.4.6's, with the entries tagged 0.4.5; neither the v0.4.4 nor the v0.4.5 tag has them, so this is a dev-branch build and the header's 0.4.5 INTRODUCED_IN blocks it. Only NeurD_load is exported, so GetProcAddress cannot discriminate, and the slot cannot be probed (a stock 0.4.4 table has 31 entries; index 33 is past it). Hence an explicit, demo-only opt-in: DXR_LEIA_LIFT_INTERACTIVE_MIN=0.4.4 (service env; clamped to >= 0.4.4) When set, the interactive availability test uses it instead of NEURD_convert_stream_dx_interactive_INTRODUCED_IN, with one WARN at activation. On a stock 0.4.4 this reads past the table and crashes — the WARN and docs say so. The NEURD_UNAVAILABLE_OUTDATED_RUNTIME fallback in the convert path is unchanged; no other gate is touched. Co-Authored-By: Claude Fable 5.1 --- docs/lift-neurd.md | 1 + src/drv_leia/leia_lift_neurd.cpp | 46 +++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index fb1a235..0349b85 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -227,6 +227,7 @@ Read once per DP at create (`leia_lift_neurd_create`). | `DXR_LEIA_LIFT_SCALE` | unset (→ stream `input_scale`, else 720p) | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. When set it overrides every stream's `input_scale`. NeurD's own default is 1440p; 720p is the fallback for latency. | | `DXR_LEIA_LIFT_VIEW_GAIN` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | | `DXR_LEIA_LIFT_CONV_GAIN` | `0.4` | `K` in the convergence map above, [−2, 2]; negative flips the sign. Calibration knob. | +| `DXR_LEIA_LIFT_INTERACTIVE_MIN` | unset (→ header, 0.4.5) | **Demo-only.** A NeurD version, e.g. `0.4.4` (clamped to ≥ 0.4.4), from which `convert_stream_dx_interactive` is trusted. For the 0.4.4 *internal-interactive* dev package, which reports 0.4.4 but carries the interactive entries. The version is the only discriminator (only `NeurD_load` is exported, and a stock 0.4.4 table is too short to probe), so on a **stock 0.4.4 this crashes** — never set it elsewhere. Read at activation; one WARN when in effect. | Under the service, remember these are read by `displayxr-service.exe`'s environment, not the client's. diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index e626364..0945ca4 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -536,6 +536,41 @@ compile_cs(ID3D11Device *dev, const char *src, const char *name) return cs; } +/*! + * DXR_LEIA_LIFT_INTERACTIVE_MIN ("0.4.4"): demo-only override of the version at + * which convert_stream_dx_interactive is trusted. The 0.4.4 internal-interactive + * NeurD package reports 0.4.4 but carries the (header: 0.4.5) interactive + * entries. Only NeurD_load is exported and a stock 0.4.4 table is too short to + * read that slot, so the version is the only discriminator — hence opt-in. + * Clamped to >= 0.4.4. Returns false when unset or unparsable. + */ +bool +interactive_min_from_env(uint64_t *out) +{ + const char *e = std::getenv("DXR_LEIA_LIFT_INTERACTIVE_MIN"); + if (e == nullptr || e[0] == '\0') { + return false; + } + // Strict "MAJ.MIN.PAT" (no sscanf: MSVC C4996). + unsigned long part[3] = {0, 0, 0}; + const char *c = e; + bool ok = true; + for (int i = 0; i < 3 && ok; i++) { + char *end = nullptr; + part[i] = std::strtoul(c, &end, 10); + ok = end != c && *end == (i < 2 ? '.' : '\0'); + c = end + (i < 2 ? 1 : 0); + } + if (!ok) { + U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN='%s' not a version (e.g. 0.4.4) — ignored", e); + return false; + } + const uint64_t floor_v = NEURD_MAKE_VERSION(0, 4, 4); + const uint64_t v = NEURD_MAKE_VERSION(part[0], part[1], part[2]); + *out = (v < floor_v) ? floor_v : v; + return true; +} + //! Drop g.nd_dev: Released only when the plug-in created it. Called with g.mtx held. void release_nd_dev_locked() @@ -732,7 +767,16 @@ activation_worker() std::lock_guard lock(g.mtx); g.backend = be; snprintf(g.backend_name, sizeof(g.backend_name), "neurd-%s", backend_str(be)); - g.interactive_unavailable = !LEIA_NEURD_HAS(nd, convert_stream_dx_interactive); + uint64_t interactive_min = NEURD_convert_stream_dx_interactive_INTRODUCED_IN; + if (interactive_min_from_env(&interactive_min)) { + U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN=%u.%u.%u — assuming interactive convert on NeurD " + "%u.%u.%u (only for the internal-interactive dev package; a stock 0.4.4 will crash here)", + (unsigned)NEURD_GET_VERSION_MAJOR(interactive_min), + (unsigned)NEURD_GET_VERSION_MINOR(interactive_min), + (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), (unsigned)NEURD_GET_VERSION_MAJOR(nd->version), + (unsigned)NEURD_GET_VERSION_MINOR(nd->version), (unsigned)NEURD_GET_VERSION_PATCH(nd->version)); + } + g.interactive_unavailable = !(nd->version >= interactive_min && nd->convert_stream_dx_interactive != nullptr); g.props_valid = false; next = setup_nd_device_locked() ? (uint32_t)G_READY : (uint32_t)G_FAILED; if (next == G_READY) { From c7290139296382e0a0a502022d958f951466ab93 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 20:44:01 -0700 Subject: [PATCH 12/21] =?UTF-8?q?fix(lift):=20DEPTH=20polarity=20=E2=80=94?= =?UTF-8?q?=20NeurD=20is=20near=3Dhigh,=20the=20spec's=20RELATIVE=20is=20l?= =?UTF-8?q?arger=3Dfarther;=20flip=20in=20the=20R8=20unpack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Panel-verified: DEPTH results came out inverted downstream (background in front in the browser's explore mode). The plug-in reports depth_semantics 0 (RELATIVE), which XR_DXR_lift defines as "larger = FARTHER, no unit", but NeurD's DEPTH output is disparity-like, near = HIGH (measured: foreground mean 236, background 1 in the R8 result). Consumers trusting the label (browser SendDepth flips RELATIVE to nearness) therefore inverted it. Flip at the bridge: kUnpackR8Cs now writes 1 - v into the R8_UNORM UAV, so the result obeys the spec. SBS/N-view unpack untouched. The convergence map uses the runtime's RELATIVE parameter, not NeurD's texture, so it is unaffected; nothing else in the plug-in reads the DEPTH texture. Known gap: the texture-result branch of stage_output (only NeurD 0.3.x's CUDA DX path) copies DEPTH as-is and is NOT flipped; commented + documented. Co-Authored-By: Claude Fable 5.1 --- docs/lift-neurd.md | 5 +++-- src/drv_leia/leia_lift_neurd.cpp | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index 0349b85..c0fbe5a 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -23,7 +23,7 @@ carries bit 8. | `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS\|NVIEW (1\|2\|4) once NeurD is present, else 0; NVIEW is dropped once READY on a NeurD without interactive convert (< 0.4.5). `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 8, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | | `lift_stream_create` | Non-blocking. Succeeds while NeurD is still activating — the NeurD stream is created lazily on the first convert. | | `lift_stream_destroy` | Releases the stream's NeurD stream and bridge resources. | -| `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device — `R8G8B8A8_UNORM` for SBS/NVIEW, `R8_UNORM` for DEPTH — owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | +| `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device — `R8G8B8A8_UNORM` for SBS/NVIEW, `R8_UNORM` for DEPTH (polarity flipped at the bridge, in the R8 unpack, from NeurD's near = high disparity to the spec's RELATIVE larger = farther) — owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | Every versioned struct is read only as far as its `struct_size` covers. `xrt_dp_lift_params.focal_px` (appended) is ignored — it only matters to a @@ -108,7 +108,8 @@ is version-gated (`LEIA_NEURD_HAS`). - **SBS and DEPTH only.** N-view needs interactive convert (0.4.5+); caps drop NVIEW once ready, and tracked-eye viewpoints are ignored (SBS uses NeurD's default pattern). Because 0.3.x hands back a texture, DEPTH arrives as `R8G8B8A8_UNORM` (copied as-is), - not `R8_UNORM`; `lift_convert`'s `out_format` reports which. + not `R8_UNORM`; `lift_convert`'s `out_format` reports which. That texture path is **not** + polarity-flipped: DEPTH stays NeurD's near = high, contrary to RELATIVE (known gap). - **Default models only** — `init_with_options` (model selection) is newer; `NeurD_init` is used instead. diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index 0945ca4..4c18292 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -307,6 +307,9 @@ void main(uint3 id : SV_DispatchThreadID) // NeurD's RGBA8 depth buffer -> single-channel R8 bridge (DEPTH contract: one // channel). NeurD writes the normalised disparity replicated into RGB; R is kept. +// Polarity: NeurD is disparity-like (near = HIGH); the lift spec's RELATIVE is +// larger = FARTHER, so the value is flipped here. Dst is an R8_UNORM UAV, so +// 1 - v stays in [0,1]. const char *kUnpackR8Cs = R"( cbuffer P : register(b0) { uint W; uint H; uint StrideBytes; uint Pad; }; RWByteAddressBuffer Src : register(u0); @@ -315,7 +318,7 @@ RWTexture2D Dst : register(u1); void main(uint3 id : SV_DispatchThreadID) { if (id.x >= W || id.y >= H) return; - Dst[id.xy] = (float)(Src.Load(id.y * StrideBytes + id.x * 4) & 0xFF) / 255.0; + Dst[id.xy] = 1.0 - (float)(Src.Load(id.y * StrideBytes + id.x * 4) & 0xFF) / 255.0; } )"; @@ -1088,6 +1091,9 @@ stage_output(lift_stream *s, const struct NeurD_image &out) ID3D11Texture2D *tex = nullptr; if (SUCCEEDED(res->QueryInterface(__uuidof(ID3D11Texture2D), (void **)&tex)) && tex != nullptr) { + // NOTE: DEPTH on this path is NOT polarity-flipped (still NeurD's + // near = high) — only kUnpackR8Cs flips. Only NeurD 0.3.x's CUDA DX + // path returns a texture. // A texture result is copied as-is (RGBA8 even for DEPTH; out_format // says so). NeurD's stream path returns buffers; this is defensive. if (!ensure_out_bridge(s, w, h, DXGI_FORMAT_R8G8B8A8_UNORM)) { @@ -1298,7 +1304,7 @@ leia_lift_neurd_get_caps(struct leia_lift_neurd *l, struct leia_lift_neurd_caps } out->max_streams = kMaxStreams; out->max_views = kMaxViews; - out->depth_semantics = 0; // relative: per-frame min-max normalised disparity + out->depth_semantics = 0; // relative: per-frame min-max normalised depth, larger = farther (NeurD's near=high disparity is flipped in kUnpackR8Cs) uint64_t lat = g.latency_ns.load(); if (s == G_READY) { // No lock: backend/backend_name are written once, before the seq_cst From e8f65d77fcaa6fee9a2a67b1b38f0f5e48680006 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 20:50:47 -0700 Subject: [PATCH 13/21] =?UTF-8?q?fix(lift):=20DXR=5FLEIA=5FLIFT=5FINTERACT?= =?UTF-8?q?IVE=5FMIN=20calls=20the=20table=20slot=20directly=20=E2=80=94?= =?UTF-8?q?=20the=20header's=20inline=20wrapper=20re-checks=200.4.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Panel result on bdb0e0f: the override activated ("interactive viewpoints available"), but the first convert logged "NeurD interactive convert unavailable" and N-view failed 5/5. The v0.4.6 header's inline NeurD_convert_stream_dx_interactive does its own NEURD_FUNC_IS_UNAVAILABLE check (version >= 0.4.5) before touching the slot, so a 0.4.4-reporting runtime got NEURD_UNAVAILABLE_OUTDATED_RUNTIME and the plug-in latched interactive_unavailable. Add nd_convert_stream_dx_interactive(): when the override admitted a runtime older than NEURD_convert_stream_dx_interactive_INTRODUCED_IN (g.interactive_direct_slot, set at activation) it calls nd->convert_stream_dx_interactive(stream, input, xyz, size, fmt, output) directly — the PFN signature from _NeurD_detail.h, i.e. the wrapper's arguments minus nd; otherwise the wrapper, so the OUTDATED_RUNTIME latch on the normal path is unchanged. Docs: the runtime polls lift_get_caps only until READY, so a convert-time latch does not remove NVIEW from the runtime's cached caps — known limit. Co-Authored-By: Claude Fable 5.1 --- docs/lift-neurd.md | 7 ++++++- src/drv_leia/leia_lift_neurd.cpp | 27 +++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index c0fbe5a..31acfe4 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -228,7 +228,7 @@ Read once per DP at create (`leia_lift_neurd_create`). | `DXR_LEIA_LIFT_SCALE` | unset (→ stream `input_scale`, else 720p) | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. When set it overrides every stream's `input_scale`. NeurD's own default is 1440p; 720p is the fallback for latency. | | `DXR_LEIA_LIFT_VIEW_GAIN` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | | `DXR_LEIA_LIFT_CONV_GAIN` | `0.4` | `K` in the convergence map above, [−2, 2]; negative flips the sign. Calibration knob. | -| `DXR_LEIA_LIFT_INTERACTIVE_MIN` | unset (→ header, 0.4.5) | **Demo-only.** A NeurD version, e.g. `0.4.4` (clamped to ≥ 0.4.4), from which `convert_stream_dx_interactive` is trusted. For the 0.4.4 *internal-interactive* dev package, which reports 0.4.4 but carries the interactive entries. The version is the only discriminator (only `NeurD_load` is exported, and a stock 0.4.4 table is too short to probe), so on a **stock 0.4.4 this crashes** — never set it elsewhere. Read at activation; one WARN when in effect. | +| `DXR_LEIA_LIFT_INTERACTIVE_MIN` | unset (→ header, 0.4.5) | **Demo-only.** A NeurD version, e.g. `0.4.4` (clamped to ≥ 0.4.4), from which `convert_stream_dx_interactive` is trusted. For the 0.4.4 *internal-interactive* dev package, which reports 0.4.4 but carries the interactive entries. The version is the only discriminator (only `NeurD_load` is exported, and a stock 0.4.4 table is too short to probe), so on a **stock 0.4.4 this crashes** — never set it elsewhere. Read at activation; one WARN when in effect. When it admits a NeurD older than 0.4.5, the plug-in calls the table slot directly (the header's inline wrapper re-checks 0.4.5). | Under the service, remember these are read by `displayxr-service.exe`'s environment, not the client's. @@ -317,5 +317,10 @@ lift-enabled runtime + this plug-in registered: - The activation thread is detached; unloading the plug-in DLL while it runs (a first activation in progress) is unsafe. The runtime does not unload plug-ins mid-session today. +- Caps are frozen once READY: the runtime (`d3d11_lift.cpp`) polls `lift_get_caps` only + while the module is not READY. If interactive convert later reports + `NEURD_UNAVAILABLE_OUTDATED_RUNTIME` at convert time, the plug-in latches it and refuses + N-view, but the runtime keeps the NVIEW bit it read at READY; N-view requests then fail + per convert rather than being rejected at stream creation. - NeurD's 32-stream limit is process-wide and shared with anything else in the service process that uses NeurD. diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index 4c18292..4ccbf8d 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -370,6 +370,10 @@ struct global uint64_t next_stream_id = 1; //! Written under g.mtx; atomic because caps reads it lock-free. std::atomic interactive_unavailable{false}; + //! DXR_LEIA_LIFT_INTERACTIVE_MIN admitted a NeurD older than the header's + //! INTRODUCED_IN: call the table slot directly (the inline wrapper would + //! re-check 0.4.5 and return OUTDATED_RUNTIME). Set at activation. + bool interactive_direct_slot = false; //! Last-applied global NeurD properties (avoid a worker round-trip per prop per frame). bool props_valid = false; @@ -574,6 +578,23 @@ interactive_min_from_env(uint64_t *out) return true; } +/*! + * NeurD_convert_stream_dx_interactive, honouring DXR_LEIA_LIFT_INTERACTIVE_MIN: + * the header's inline wrapper re-checks version >= INTRODUCED_IN (0.4.5), so + * when the override admitted an older-reporting runtime the table slot is + * called directly (same arguments minus @p nd). Called with g.mtx held. + */ +enum NeurD_status +nd_convert_stream_dx_interactive(struct NeurD const *nd, struct NeurD_stream *stream, struct NeurD_image const *input, + const float *viewpoints_xyz, int size, enum NeurD_pixel_format output_pix_fmt, + struct NeurD_image *output) +{ + if (g.interactive_direct_slot && nd->convert_stream_dx_interactive != nullptr) { + return nd->convert_stream_dx_interactive(stream, input, viewpoints_xyz, size, output_pix_fmt, output); + } + return NeurD_convert_stream_dx_interactive(nd, stream, input, viewpoints_xyz, size, output_pix_fmt, output); +} + //! Drop g.nd_dev: Released only when the plug-in created it. Called with g.mtx held. void release_nd_dev_locked() @@ -780,6 +801,8 @@ activation_worker() (unsigned)NEURD_GET_VERSION_MINOR(nd->version), (unsigned)NEURD_GET_VERSION_PATCH(nd->version)); } g.interactive_unavailable = !(nd->version >= interactive_min && nd->convert_stream_dx_interactive != nullptr); + g.interactive_direct_slot = !g.interactive_unavailable.load() && + nd->version < NEURD_convert_stream_dx_interactive_INTRODUCED_IN; g.props_valid = false; next = setup_nd_device_locked() ? (uint32_t)G_READY : (uint32_t)G_FAILED; if (next == G_READY) { @@ -1642,8 +1665,8 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, struct NeurD_image nout = {}; enum NeurD_status st; if (!vp.empty() && !g.interactive_unavailable) { - st = NeurD_convert_stream_dx_interactive(nd, s->ns, &nin, vp.data(), (int)vp.size(), - NEURD_PIXEL_FORMAT_RGBA8, &nout); + st = nd_convert_stream_dx_interactive(nd, s->ns, &nin, vp.data(), (int)vp.size(), + NEURD_PIXEL_FORMAT_RGBA8, &nout); if (st == NEURD_UNAVAILABLE_OUTDATED_RUNTIME) { g.interactive_unavailable = true; U_LOG_W("Leia lift: NeurD interactive convert unavailable — tracked viewpoints ignored"); From 9b7af2e79662572ff8cb10416d5b88333f387f68 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 25 Sep 2026 21:13:24 -0700 Subject: [PATCH 14/21] =?UTF-8?q?fix(lift):=20NeurD=20cannot=20tile=20beyo?= =?UTF-8?q?nd=20SBS=20through=20the=20public=20API=20=E2=80=94=20drop=20NV?= =?UTF-8?q?IEW=20from=20caps,=20derive=20view=5Fcount=20from=20the=20outpu?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Panel result on bd6b52b: an N-view probe (--views 4) came back as a plain 2-tile SBS (4096x1362) stamped view_count=4. In media_sdk (v0.4.6, same as the dev build) Config::adjust() runs on every convert via adjust_cfg_for_image() — including the interactive path — and re-derives tiled_cols/rows from output_type (SBS 2x1, TB 1x2, DEPTH 1x1), and NeurD_set_prop_1i(OUTPUT_TYPE) accepts only 0..2. So OUTPUT_TILES_W/H are dead through the public API and N > 2 tiles per convert is impossible until a tiled output type exists. - caps: never advertise NVIEW (pre-READY or READY). - stream_create(NVIEW): refused with a WARN naming the reason. - convert: check NeurD's actual layout (output aspect / input aspect = tile count) against the mode's expected tiles and drop a mismatching frame, so a result can never be delivered under the wrong label. - Interactive SBS viewpoint path unchanged. - docs: N-view section + asks for Leia media_sdk. Co-Authored-By: Claude Fable 5.1 --- docs/lift-neurd.md | 42 +++++++++++++++++++++++--------- src/drv_leia/leia_lift_neurd.cpp | 33 ++++++++++++++++++++----- 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index 31acfe4..be1e081 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -20,10 +20,10 @@ carries bit 8. | Slot | Plug-in behaviour | |---|---| -| `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS\|NVIEW (1\|2\|4) once NeurD is present, else 0; NVIEW is dropped once READY on a NeurD without interactive convert (< 0.4.5). `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 8, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | +| `lift_get_caps` | Non-blocking. `modes` = DEPTH\|SBS (1\|2) once NeurD is present, else 0. **Never NVIEW**: not available on NeurD ≤ 0.4.6 (public output types fix the tile grid); SBS with explicit viewpoints only. `state` 0 unavailable / 1 activating / 2 ready. `max_streams` 32 (NeurD's process limit), `max_views` 8, `depth_semantics` 0 (relative), `backend` e.g. `neurd-directml`, `typical_latency_ns` = measured EMA (prior: 22 ms DirectML, 14 ms CUDA). | | `lift_stream_create` | Non-blocking. Succeeds while NeurD is still activating — the NeurD stream is created lazily on the first convert. | | `lift_stream_destroy` | Releases the stream's NeurD stream and bridge resources. | -| `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device — `R8G8B8A8_UNORM` for SBS/NVIEW, `R8_UNORM` for DEPTH (polarity flipped at the bridge, in the R8 unpack, from NeurD's near = high disparity to the spec's RELATIVE larger = farther) — owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | +| `lift_convert` | **Synchronous, blocking** (≈ bridge + inference). Returns an `ID3D11Texture2D*` on the caller's device — `R8G8B8A8_UNORM` for SBS, `R8_UNORM` for DEPTH (polarity flipped at the bridge, in the R8 unpack, from NeurD's near = high disparity to the spec's RELATIVE larger = farther) — owned by the stream and valid until the next convert on that stream. Returns false while activating/unavailable. | Every versioned struct is read only as far as its `struct_size` covers. `xrt_dp_lift_params.focal_px` (appended) is ignored — it only matters to a @@ -42,8 +42,10 @@ unless called), which is what a runtime without the lift factory falls back to. Output layouts (runtime contract): DEPTH = one channel at the inference resolution (NeurD writes the normalised disparity replicated into RGB; the unpack keeps R); -SBS = 2 views side by side; NVIEW = `view_count` views side by side in **one row**, -view 0 leftmost. Tile size = the *inference* resolution (after autoscaling), not the +SBS = 2 views side by side. (NVIEW — `view_count` views in one row — is not offered +by this module; see *N-view* below.) Every convert checks NeurD's actual layout +(output aspect ÷ input aspect = tile count) against the mode and drops a frame that +doesn't match, so a result can never be mislabelled. Tile size = the *inference* resolution (after autoscaling), not the input size — use `out_w/out_h`. One row is why `max_views` is 8: 8 × 2560 (1440p) is the widest that fits D3D11's 16384-texel limit; a wider request fails with a WARN. @@ -105,8 +107,8 @@ is version-gated (`LEIA_NEURD_HAS`). `DXR_LEIA_LIFT_BACKEND=cuda` in the **service's** environment (`displayxr-service.exe` reads its own env, not the client's). The default adapter must be the NVIDIA GPU for the interop to bind — on a hybrid box, pin the service to the dGPU. -- **SBS and DEPTH only.** N-view needs interactive convert (0.4.5+); caps drop NVIEW once - ready, and tracked-eye viewpoints are ignored (SBS uses NeurD's default pattern). +- **SBS and DEPTH only** (as on every NeurD today — see *N-view*); without interactive + convert (0.4.5+) tracked-eye viewpoints are ignored (SBS uses NeurD's default pattern). Because 0.3.x hands back a texture, DEPTH arrives as `R8G8B8A8_UNORM` (copied as-is), not `R8_UNORM`; `lift_convert`'s `out_format` reports which. That texture path is **not** polarity-flipped: DEPTH stays NeurD's near = high, contrary to RELATIVE (known gap). @@ -185,7 +187,25 @@ Viewpoint source, in order: For N views from an eye pair, views are spaced one eye baseline apart and centred on the eye midpoint. If NeurD predates the interactive API (< 0.4.5), SBS falls back to the -default pattern and NVIEW is refused. +default pattern. (The N-view paths are unreachable today: NVIEW streams are refused.) + +## N-view + +**Not available on NeurD ≤ 0.4.6.** NeurD's `Config::adjust()` runs on every convert +(including the interactive one) and re-derives the tile grid from the output type — +SBS → 2×1, TB → 1×2, DEPTH → 1×1 — and the public `NEURD_PROP_OUTPUT_TYPE` accepts only +those, so `OUTPUT_TILES_W/H` are overridden. Panel-verified: a 4-view request came back +as a plain 2-tile SBS. Caps never advertise NVIEW, and `lift_stream_create(NVIEW)` fails +with a WARN. What works: SBS with explicit viewpoints (two, from the runtime or tracked +eyes). + +### Asks for Leia media_sdk + +- A tiled / multiview output type usable with `convert_stream_dx_interactive` + (N viewpoints → N tiles in one convert). +- A public release of the 0.4.4 *internal-interactive* build (interactive entries in a + build whose reported version matches its table). +- A working 0.4.6 package. ## Parameters @@ -195,7 +215,7 @@ default pattern and NVIEW is refused. | `convergence` in [0, 1] | `AUTO_CONVERGENCE = FALSE`, `CONVERGENCE = clamp(K · (c − 0.5), ±0.2)`, K = `DXR_LEIA_LIFT_CONV_GAIN` | | `strength` | `GAIN_MULTIPLIER` (1 = NeurD's calibrated budget, 0 = flat), clamped [0, 10]; negative → 1.0 | | `inpaint` | NeurD always fills disocclusions: 0 → `V1_STRETCH` (cheapest), non-zero → `V1_BLUR` (the DX video path supports V1 only) | -| `view_count` | NVIEW row width (2..8) | +| `view_count` | NVIEW row width — unused (NVIEW not offered on NeurD) | **Convergence calibration.** The runtime's `convergence` is the relative depth placed at the display plane, normalised to [0, 1] over the frame's depth range (0 = nearest @@ -318,9 +338,7 @@ lift-enabled runtime + this plug-in registered: (a first activation in progress) is unsafe. The runtime does not unload plug-ins mid-session today. - Caps are frozen once READY: the runtime (`d3d11_lift.cpp`) polls `lift_get_caps` only - while the module is not READY. If interactive convert later reports - `NEURD_UNAVAILABLE_OUTDATED_RUNTIME` at convert time, the plug-in latches it and refuses - N-view, but the runtime keeps the NVIEW bit it read at READY; N-view requests then fail - per convert rather than being rejected at stream creation. + while the module is not READY. A convert-time `NEURD_UNAVAILABLE_OUTDATED_RUNTIME` + latch (SBS then falls back to the default pattern) is therefore not reflected in caps. - NeurD's 32-stream limit is process-wide and shared with anything else in the service process that uses NeurD. diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index 4ccbf8d..0320784 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -1319,12 +1319,11 @@ leia_lift_neurd_get_caps(struct leia_lift_neurd *l, struct leia_lift_neurd_caps if (out->state == LEIA_LIFT_STATE_UNAVAILABLE) { return true; } + // Never NVIEW: NeurD's Config::adjust() re-derives the tile grid from the + // output type on every convert (SBS 2x1, TB 1x2, DEPTH 1x1), and the public + // OUTPUT_TYPE prop accepts only 0..2, so OUTPUT_TILES_W/H are overridden — + // N > 2 tiles per convert is impossible through the public API (<= 0.4.6). out->modes = LEIA_LIFT_MODE_DEPTH | LEIA_LIFT_MODE_SBS; - // N-view needs interactive convert (NeurD 0.4.5+). Known only once READY; - // before that, advertise it optimistically (it may still turn out absent). - if (s != G_READY || !g.interactive_unavailable.load()) { - out->modes |= LEIA_LIFT_MODE_NVIEW; - } out->max_streams = kMaxStreams; out->max_views = kMaxViews; out->depth_semantics = 0; // relative: per-frame min-max normalised depth, larger = farther (NeurD's near=high disparity is flipped in kUnpackR8Cs) @@ -1356,7 +1355,12 @@ leia_lift_neurd_stream_create(struct leia_lift_neurd *l, const struct leia_lift_ if (l == nullptr || desc == nullptr || out_id == nullptr || !l->k.enabled) { return false; } - if (desc->mode != LEIA_LIFT_MODE_DEPTH && desc->mode != LEIA_LIFT_MODE_SBS && desc->mode != LEIA_LIFT_MODE_NVIEW) { + if (desc->mode == LEIA_LIFT_MODE_NVIEW) { + LIFT_WARN_ONCE("Leia lift: N-view not supported — NeurD public API tiles are fixed by output type; " + "N-view needs a tiled output type (ask Leia)"); + return false; + } + if (desc->mode != LEIA_LIFT_MODE_DEPTH && desc->mode != LEIA_LIFT_MODE_SBS) { LIFT_WARN_ONCE("Leia lift: stream_create with unsupported mode %u", desc->mode); return false; } @@ -1687,6 +1691,23 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, return false; } + // ---- 5b. Trust NeurD's actual layout, not the requested one: tiles in + // the one-row output = output aspect / input aspect (robust to NeurD's + // inference autoscaling). A mismatch would be mislabelled downstream. + if (nout.width <= 0 || nout.height <= 0) { + LIFT_WARN_ONCE("Leia lift: NeurD output has no size (%dx%d)", nout.width, nout.height); + return false; + } + const double tiles_f = + ((double)nout.width * (double)h) / ((double)nout.height * (double)w); + const uint32_t tiles = (uint32_t)(tiles_f + 0.5); + if (tiles != cols * rows) { + LIFT_WARN_ONCE("Leia lift: NeurD returned %dx%d (%u tile(s)) for a %ux%u input, expected %u — " + "frame dropped", + nout.width, nout.height, tiles, w, h, cols * rows); + return false; + } + // ---- 6. Output -> bridge (NeurD device), drained; hand ours back. if (!stage_output(s, nout) || !drain(g.nd_ctx, g.nd_done, "NeurD (unpack)")) { return false; From a4468c0e68d7ea68787fb1d7994164625d62e4ac Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 01:48:39 -0700 Subject: [PATCH 15/21] =?UTF-8?q?feat(lift):=20lift=20knobs=20from=20HKLM\?= =?UTF-8?q?SOFTWARE\DisplayXR\Leia\Lift=20(env=20still=20wins)=20=E2=80=94?= =?UTF-8?q?=20a=20respawned=20service=20no=20longer=20loses=20them=20silen?= =?UTF-8?q?tly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lift knobs lived only in displayxr-service.exe's environment. Any respawn (tray relaunch, HKLM Run at logon, crash restart) starts the service with the logon env, and the knobs vanished silently — the only tell was "interactive viewpoints UNAVAILABLE". Registry gates, not env vars. - Each knob now resolves env > HKLM\SOFTWARE\DisplayXR\Leia\Lift (REG_SZ, opened with KEY_WOW64_64KEY) > default. Values: Backend, InteractiveMin, Scale, ViewGain, ConvGain — same grammar as DXR_LEIA_LIFT_BACKEND / _INTERACTIVE_MIN / _SCALE / _VIEW_GAIN / _CONV_GAIN. DXR_LEIA_LIFT (the kill switch) stays env-only. - One parser per knob shared by both sources (knob_lookup returns the raw UTF-8 string + its source; parse_backend/parse_scale/parse_float_in/ parse_interactive_min own the grammar). InteractiveMin is now read with the other knobs at DP create instead of ad hoc at activation. - kick() takes the first acquirer's knobs; activation logs ONE WARN with every knob's effective value and source. - scripts/set-lift-knobs.bat (elevated): Backend=directml + InteractiveMin=0.4.4, --clear deletes the key. - docs: Registry value column, why, exact reg add lines. No behaviour change with no key and no env. The gain knobs now parse strictly (strtof, whole string) where atof accepted trailing junk. Co-Authored-By: Claude Fable 5.1 --- docs/lift-neurd.md | 48 +++-- scripts/set-lift-knobs.bat | 47 +++++ src/drv_leia/leia_lift_neurd.cpp | 323 ++++++++++++++++++++++--------- 3 files changed, 315 insertions(+), 103 deletions(-) create mode 100644 scripts/set-lift-knobs.bat diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index be1e081..d4db23c 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -104,8 +104,8 @@ is version-gated (`LEIA_NEURD_HAS`). same device, so the bridge is otherwise unchanged. One WARN names the case and the LUID. - **CUDA backend required for the DX path on 0.3.x.** Its DX convert is implemented on CUDA↔D3D11 interop; the DirectML build's DX path is not verified. Set - `DXR_LEIA_LIFT_BACKEND=cuda` in the **service's** environment (`displayxr-service.exe` - reads its own env, not the client's). The default adapter must be the NVIDIA GPU for + backend `cuda` — `DXR_LEIA_LIFT_BACKEND=cuda` in the **service's** environment, or + better the `Backend` registry value (see *Knobs*), which survives service respawns. The default adapter must be the NVIDIA GPU for the interop to bind — on a hybrid box, pin the service to the dGPU. - **SBS and DEPTH only** (as on every NeurD today — see *N-view*); without interactive convert (0.4.5+) tracked-eye viewpoints are ignored (SBS uses NeurD's default pattern). @@ -239,19 +239,39 @@ inference resolution. ## Knobs -Read once per DP at create (`leia_lift_neurd_create`). +Read once per DP at create (`leia_lift_neurd_create`). Each knob comes from the +**environment**, else the **registry** (`HKLM\SOFTWARE\DisplayXR\Leia\Lift`, REG_SZ, +64-bit view, same grammar as the env var), else its default — env > registry > default. +At activation one WARN lists every knob's effective value and where it came from: -| Env | Default | Effect | -|---|---|---| -| `DXR_LEIA_LIFT` | on | `0` / `off` → caps `modes=0, state=0`; NeurD is never probed or loaded. | -| `DXR_LEIA_LIFT_BACKEND` | `directml` | `auto` \| `directml` \| `cuda` \| `openvino`. First activation in the process wins (NeurD's forced backend is sticky). On NeurD 0.4.3+ only DirectML yields a D3D11 device; on 0.3.x use `cuda` (see *NeurD 0.3.x*). | -| `DXR_LEIA_LIFT_SCALE` | unset (→ stream `input_scale`, else 720p) | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. When set it overrides every stream's `input_scale`. NeurD's own default is 1440p; 720p is the fallback for latency. | -| `DXR_LEIA_LIFT_VIEW_GAIN` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | -| `DXR_LEIA_LIFT_CONV_GAIN` | `0.4` | `K` in the convergence map above, [−2, 2]; negative flips the sign. Calibration knob. | -| `DXR_LEIA_LIFT_INTERACTIVE_MIN` | unset (→ header, 0.4.5) | **Demo-only.** A NeurD version, e.g. `0.4.4` (clamped to ≥ 0.4.4), from which `convert_stream_dx_interactive` is trusted. For the 0.4.4 *internal-interactive* dev package, which reports 0.4.4 but carries the interactive entries. The version is the only discriminator (only `NeurD_load` is exported, and a stock 0.4.4 table is too short to probe), so on a **stock 0.4.4 this crashes** — never set it elsewhere. Read at activation; one WARN when in effect. When it admits a NeurD older than 0.4.5, the plug-in calls the table slot directly (the header's inline wrapper re-checks 0.4.5). | - -Under the service, remember these are read by `displayxr-service.exe`'s environment, -not the client's. +``` +Leia lift: knobs backend=directml(reg) interactive_min=0.4.4(reg) scale=per-stream(default) view_gain=1.00(default) conv_gain=0.40(default) [env > HKLM\SOFTWARE\DisplayXR\Leia\Lift > default] +``` + +**Why the registry.** Under the service the knobs are read from `displayxr-service.exe`'s +environment, not the client's. Any respawn — tray relaunch, the HKLM `Run` key at logon, +a crash restart — starts the service with the logon environment, and env-only knobs +vanish silently (the only tell was "interactive viewpoints UNAVAILABLE"). A registry +value survives every respawn. Env still wins when set, so `.bat`-driven runs behave as +before; with neither set, nothing changes. + +| Env | Registry value | Default | Effect | +|---|---|---|---| +| `DXR_LEIA_LIFT` | — (env only) | on | `0` / `off` → caps `modes=0, state=0`; NeurD is never probed or loaded. | +| `DXR_LEIA_LIFT_BACKEND` | `Backend` | `directml` | `auto` \| `directml` \| `cuda` \| `openvino`. First activation in the process wins (NeurD's forced backend is sticky). On NeurD 0.4.3+ only DirectML yields a D3D11 device; on 0.3.x use `cuda` (see *NeurD 0.3.x*). | +| `DXR_LEIA_LIFT_SCALE` | `Scale` | unset (→ stream `input_scale`, else 720p) | Inference height bucket: `720` \| `1080` \| `1440` \| `none`. When set it overrides every stream's `input_scale`. NeurD's own default is 1440p; 720p is the fallback for latency. | +| `DXR_LEIA_LIFT_VIEW_GAIN` | `ViewGain` | `1.0` | `G` in the eye → viewpoint mapping above, [0, 10]. | +| `DXR_LEIA_LIFT_CONV_GAIN` | `ConvGain` | `0.4` | `K` in the convergence map above, [−2, 2]; negative flips the sign. Calibration knob. | +| `DXR_LEIA_LIFT_INTERACTIVE_MIN` | `InteractiveMin` | unset (→ header, 0.4.5) | **Demo-only.** A NeurD version, e.g. `0.4.4` (clamped to ≥ 0.4.4), from which `convert_stream_dx_interactive` is trusted. For the 0.4.4 *internal-interactive* dev package, which reports 0.4.4 but carries the interactive entries. The version is the only discriminator (only `NeurD_load` is exported, and a stock 0.4.4 table is too short to probe), so on a **stock 0.4.4 this crashes** — never set it elsewhere. One extra WARN when in effect. When it admits a NeurD older than 0.4.5, the plug-in calls the table slot directly (the header's inline wrapper re-checks 0.4.5). | + +The two demo knobs, set once on the box (elevated prompt), then restart the service: + +```bat +reg add "HKLM\SOFTWARE\DisplayXR\Leia\Lift" /v Backend /t REG_SZ /d directml /f /reg:64 +reg add "HKLM\SOFTWARE\DisplayXR\Leia\Lift" /v InteractiveMin /t REG_SZ /d 0.4.4 /f /reg:64 +``` + +`scripts\set-lift-knobs.bat` does exactly this (`--clear` deletes the key). ## Building diff --git a/scripts/set-lift-knobs.bat b/scripts/set-lift-knobs.bat new file mode 100644 index 0000000..d35ccaf --- /dev/null +++ b/scripts/set-lift-knobs.bat @@ -0,0 +1,47 @@ +@echo off +setlocal + +:: ============================================================ +:: Set the Leia lift demo knobs in the registry (run ELEVATED). +:: ============================================================ +:: Usage: scripts\set-lift-knobs.bat write Backend=directml, InteractiveMin=0.4.4 +:: scripts\set-lift-knobs.bat --clear delete HKLM\SOFTWARE\DisplayXR\Leia\Lift +:: +:: The lift reads each knob from the env var first (DXR_LEIA_LIFT_*), else from +:: this key, else its default. The registry survives displayxr-service.exe +:: respawns (tray relaunch, HKLM Run at logon, crash restart), which start with +:: the logon environment and would otherwise drop env-only knobs silently. +:: InteractiveMin=0.4.4 is ONLY for the NeurD 0.4.4 internal-interactive package; +:: on a stock NeurD 0.4.4 it crashes the service. See docs\lift-neurd.md. +:: Restart displayxr-service.exe afterwards for the change to take effect. + +set "KEY=HKLM\SOFTWARE\DisplayXR\Leia\Lift" + +net session >nul 2>&1 +if errorlevel 1 ( + echo ERROR: run this from an elevated ^(Administrator^) prompt. + exit /b 1 +) + +if /i "%~1"=="--clear" ( + reg query "%KEY%" /reg:64 >nul 2>&1 + if errorlevel 1 ( + echo %KEY% not present - nothing to clear. + exit /b 0 + ) + reg delete "%KEY%" /f /reg:64 || exit /b 1 + echo Deleted %KEY%. + exit /b 0 +) + +if not "%~1"=="" ( + echo Usage: %~nx0 [--clear] + exit /b 2 +) + +reg add "%KEY%" /v Backend /t REG_SZ /d directml /f /reg:64 || exit /b 1 +reg add "%KEY%" /v InteractiveMin /t REG_SZ /d 0.4.4 /f /reg:64 || exit /b 1 +echo Wrote %KEY%: Backend=directml InteractiveMin=0.4.4 +reg query "%KEY%" /reg:64 +echo Restart displayxr-service.exe for the lift to pick these up. +exit /b 0 diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index 0320784..e38dbc4 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -173,7 +173,11 @@ backend_str(enum NeurD_backend b) /* * - * Env knobs (read once per DP handle, at leia_lift_neurd_create). + * Knobs (read once per DP handle, at leia_lift_neurd_create). Each comes from + * the environment, else HKLM\SOFTWARE\DisplayXR\Leia\Lift (REG_SZ, same + * grammar), else its default. The registry exists because a respawned service + * (tray relaunch, HKLM Run at logon, crash restart) starts with the logon env + * and would otherwise lose env-only knobs silently. * */ @@ -185,14 +189,33 @@ enum backend_choice BACKEND_OPENVINO = NEURD_BACKEND_OPENVINO, }; +enum knob_src +{ + KSRC_DEFAULT = 0, + KSRC_ENV, + KSRC_REG, +}; + +const char * +knob_src_str(knob_src s) +{ + switch (s) { + case KSRC_ENV: return "env"; + case KSRC_REG: return "reg"; + default: return "default"; + } +} + struct knobs { - bool enabled; //!< DXR_LEIA_LIFT (default on) - int backend; //!< DXR_LEIA_LIFT_BACKEND (default directml) - int32_t autoscaling; //!< DXR_LEIA_LIFT_SCALE (default 720p) - bool scale_forced; //!< DXR_LEIA_LIFT_SCALE was set: overrides the stream's input_scale - float view_gain; //!< DXR_LEIA_LIFT_VIEW_GAIN (default 1.0) - float conv_gain; //!< DXR_LEIA_LIFT_CONV_GAIN (default 0.4): relative convergence -> NeurD units + bool enabled; //!< DXR_LEIA_LIFT (default on; env only) + int backend; //!< DXR_LEIA_LIFT_BACKEND / Backend (default directml) + int32_t autoscaling; //!< DXR_LEIA_LIFT_SCALE / Scale (default 720p) + bool scale_forced; //!< Scale was set: overrides the stream's input_scale + float view_gain; //!< DXR_LEIA_LIFT_VIEW_GAIN / ViewGain (default 1.0) + float conv_gain; //!< DXR_LEIA_LIFT_CONV_GAIN / ConvGain (default 0.4): relative convergence -> NeurD units + uint64_t interactive_min; //!< DXR_LEIA_LIFT_INTERACTIVE_MIN / InteractiveMin; 0 = unset (header's INTRODUCED_IN) + knob_src src_backend, src_scale, src_view_gain, src_conv_gain, src_interactive_min; }; bool @@ -201,6 +224,136 @@ env_ieq(const char *a, const char *b) return _stricmp(a, b) == 0; } +const wchar_t *const kLiftRegKey = L"SOFTWARE\\DisplayXR\\Leia\\Lift"; + +/*! + * One knob's raw string: env @p env_name if set and non-empty, else REG_SZ + * @p reg_name under @p reg (may be NULL), else absent. @p out is UTF-8. + */ +knob_src +knob_lookup(HKEY reg, const char *env_name, const wchar_t *reg_name, char *out, size_t cap) +{ + out[0] = '\0'; + const char *e = std::getenv(env_name); + if (e != nullptr && e[0] != '\0') { + snprintf(out, cap, "%s", e); + return KSRC_ENV; + } + if (reg == nullptr) { + return KSRC_DEFAULT; + } + wchar_t w[128] = {}; + DWORD cb = sizeof(w); // bytes, incl. the terminator RegGetValueW guarantees + LSTATUS r = RegGetValueW(reg, nullptr, reg_name, RRF_RT_REG_SZ, nullptr, w, &cb); + if (r == ERROR_FILE_NOT_FOUND) { + return KSRC_DEFAULT; + } + if (r != ERROR_SUCCESS) { + U_LOG_W("Leia lift: HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift\\%ls unreadable (error %ld, REG_SZ <= 127 chars " + "expected) — ignored", + reg_name, (long)r); + return KSRC_DEFAULT; + } + if (w[0] == L'\0') { + return KSRC_DEFAULT; + } + int n = WideCharToMultiByte(CP_UTF8, 0, w, -1, out, (int)cap, nullptr, nullptr); + if (n <= 0) { + out[0] = '\0'; + return KSRC_DEFAULT; + } + return KSRC_REG; +} + +// ---- One grammar per knob, shared by both sources. Each returns false on a +// value it does not recognise (caller WARNs and keeps the default). + +bool +parse_backend(const char *v, int *out) +{ + if (env_ieq(v, "auto")) { + *out = BACKEND_AUTO; + } else if (env_ieq(v, "cuda")) { + *out = BACKEND_CUDA; + } else if (env_ieq(v, "directml") || env_ieq(v, "dml")) { + *out = BACKEND_DIRECTML; + } else if (env_ieq(v, "openvino")) { + *out = BACKEND_OPENVINO; + } else { + return false; + } + return true; +} + +bool +parse_scale(const char *v, int32_t *out) +{ + if (env_ieq(v, "none") || env_ieq(v, "native") || env_ieq(v, "0")) { + *out = NEURD_INPUT_AUTOSCALING_NONE; + } else if (env_ieq(v, "720") || env_ieq(v, "720p")) { + *out = NEURD_INPUT_AUTOSCALING_720P; + } else if (env_ieq(v, "1080") || env_ieq(v, "1080p")) { + *out = NEURD_INPUT_AUTOSCALING_1080P; + } else if (env_ieq(v, "1440") || env_ieq(v, "1440p")) { + *out = NEURD_INPUT_AUTOSCALING_1440P; + } else { + return false; + } + return true; +} + +bool +parse_float_in(const char *v, float lo, float hi, float *out) +{ + char *end = nullptr; + float f = std::strtof(v, &end); + if (end == v || *end != '\0' || !std::isfinite(f) || f < lo || f > hi) { + return false; + } + *out = f; + return true; +} + +//! Strict "MAJ.MIN.PAT" (no sscanf: MSVC C4996), clamped to >= 0.4.4. +bool +parse_interactive_min(const char *v, uint64_t *out) +{ + unsigned long part[3] = {0, 0, 0}; + const char *c = v; + bool ok = true; + for (int i = 0; i < 3 && ok; i++) { + char *end = nullptr; + part[i] = std::strtoul(c, &end, 10); + ok = end != c && *end == (i < 2 ? '.' : '\0'); + c = end + (i < 2 ? 1 : 0); + } + if (!ok) { + return false; + } + const uint64_t floor_v = NEURD_MAKE_VERSION(0, 4, 4); + const uint64_t ver = NEURD_MAKE_VERSION(part[0], part[1], part[2]); + *out = (ver < floor_v) ? floor_v : ver; + return true; +} + +const char * +scale_str(int32_t a) +{ + switch (a) { + case NEURD_INPUT_AUTOSCALING_NONE: return "none"; + case NEURD_INPUT_AUTOSCALING_720P: return "720"; + case NEURD_INPUT_AUTOSCALING_1080P: return "1080"; + case NEURD_INPUT_AUTOSCALING_1440P: return "1440"; + default: return "?"; + } +} + +const char * +backend_choice_str(int b) +{ + return b == BACKEND_AUTO ? "auto" : backend_str((enum NeurD_backend)b); +} + struct knobs read_knobs() { @@ -210,63 +363,75 @@ read_knobs() k.autoscaling = NEURD_INPUT_AUTOSCALING_720P; k.view_gain = 1.0f; k.conv_gain = 0.4f; + k.interactive_min = 0; const char *e = std::getenv("DXR_LEIA_LIFT"); if (e != nullptr && (e[0] == '0' || env_ieq(e, "off") || env_ieq(e, "false"))) { k.enabled = false; } - e = std::getenv("DXR_LEIA_LIFT_BACKEND"); - if (e != nullptr && e[0] != '\0') { - if (env_ieq(e, "auto")) { - k.backend = BACKEND_AUTO; - } else if (env_ieq(e, "cuda")) { - k.backend = BACKEND_CUDA; - } else if (env_ieq(e, "directml") || env_ieq(e, "dml")) { - k.backend = BACKEND_DIRECTML; - } else if (env_ieq(e, "openvino")) { - k.backend = BACKEND_OPENVINO; + // Explicit 64-bit view: the service is 64-bit, but a WOW64 host must not + // be redirected to Wow6432Node. + HKEY reg = nullptr; + if (RegOpenKeyExW(HKEY_LOCAL_MACHINE, kLiftRegKey, 0, KEY_QUERY_VALUE | KEY_WOW64_64KEY, ®) != ERROR_SUCCESS) { + reg = nullptr; + } + + char v[256]; + knob_src src; + + src = knob_lookup(reg, "DXR_LEIA_LIFT_BACKEND", L"Backend", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_backend(v, &k.backend)) { + k.src_backend = src; } else { - U_LOG_W("Leia lift: DXR_LEIA_LIFT_BACKEND='%s' not recognised (auto|directml|cuda|openvino) — " - "using directml", - e); + U_LOG_W("Leia lift: backend '%s' (%s) not recognised (auto|directml|cuda|openvino) — using directml", v, + knob_src_str(src)); } } - e = std::getenv("DXR_LEIA_LIFT_SCALE"); - if (e != nullptr && e[0] != '\0') { - k.scale_forced = true; - if (env_ieq(e, "none") || env_ieq(e, "native") || env_ieq(e, "0")) { - k.autoscaling = NEURD_INPUT_AUTOSCALING_NONE; - } else if (env_ieq(e, "720") || env_ieq(e, "720p")) { - k.autoscaling = NEURD_INPUT_AUTOSCALING_720P; - } else if (env_ieq(e, "1080") || env_ieq(e, "1080p")) { - k.autoscaling = NEURD_INPUT_AUTOSCALING_1080P; - } else if (env_ieq(e, "1440") || env_ieq(e, "1440p")) { - k.autoscaling = NEURD_INPUT_AUTOSCALING_1440P; + src = knob_lookup(reg, "DXR_LEIA_LIFT_SCALE", L"Scale", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + k.scale_forced = true; // as before: a set-but-unrecognised value still forces 720 + if (parse_scale(v, &k.autoscaling)) { + k.src_scale = src; } else { - U_LOG_W("Leia lift: DXR_LEIA_LIFT_SCALE='%s' not recognised (720|1080|1440|none) — using 720", e); + U_LOG_W("Leia lift: scale '%s' (%s) not recognised (720|1080|1440|none) — using 720", v, + knob_src_str(src)); } } - e = std::getenv("DXR_LEIA_LIFT_VIEW_GAIN"); - if (e != nullptr && e[0] != '\0') { - float g = (float)std::atof(e); - if (std::isfinite(g) && g >= 0.0f && g <= 10.0f) { - k.view_gain = g; + src = knob_lookup(reg, "DXR_LEIA_LIFT_VIEW_GAIN", L"ViewGain", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_float_in(v, 0.0f, 10.0f, &k.view_gain)) { + k.src_view_gain = src; } else { - U_LOG_W("Leia lift: DXR_LEIA_LIFT_VIEW_GAIN='%s' out of range [0,10] — using 1.0", e); + U_LOG_W("Leia lift: view gain '%s' (%s) not a number in [0,10] — using 1.0", v, knob_src_str(src)); } } - e = std::getenv("DXR_LEIA_LIFT_CONV_GAIN"); - if (e != nullptr && e[0] != '\0') { - float cg = (float)std::atof(e); - if (std::isfinite(cg) && cg >= -2.0f && cg <= 2.0f) { - k.conv_gain = cg; + + src = knob_lookup(reg, "DXR_LEIA_LIFT_CONV_GAIN", L"ConvGain", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_float_in(v, -2.0f, 2.0f, &k.conv_gain)) { + k.src_conv_gain = src; + } else { + U_LOG_W("Leia lift: conv gain '%s' (%s) not a number in [-2,2] — using 0.4", v, knob_src_str(src)); + } + } + + src = knob_lookup(reg, "DXR_LEIA_LIFT_INTERACTIVE_MIN", L"InteractiveMin", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_interactive_min(v, &k.interactive_min)) { + k.src_interactive_min = src; } else { - U_LOG_W("Leia lift: DXR_LEIA_LIFT_CONV_GAIN='%s' out of range [-2,2] — using 0.4", e); + U_LOG_W("Leia lift: interactive min '%s' (%s) not a version (e.g. 0.4.4) — ignored", v, + knob_src_str(src)); } } + + if (reg != nullptr) { + RegCloseKey(reg); + } return k; } @@ -350,6 +515,7 @@ struct global int requested_backend = BACKEND_DIRECTML; //!< First acquirer's choice (NeurD's forced backend is sticky). int32_t default_autoscaling = NEURD_INPUT_AUTOSCALING_720P; + struct knobs k0 = {}; //!< First acquirer's knobs (logged + interactive_min at activation). HMODULE lib = nullptr; struct NeurD const *nd = nullptr; @@ -543,41 +709,6 @@ compile_cs(ID3D11Device *dev, const char *src, const char *name) return cs; } -/*! - * DXR_LEIA_LIFT_INTERACTIVE_MIN ("0.4.4"): demo-only override of the version at - * which convert_stream_dx_interactive is trusted. The 0.4.4 internal-interactive - * NeurD package reports 0.4.4 but carries the (header: 0.4.5) interactive - * entries. Only NeurD_load is exported and a stock 0.4.4 table is too short to - * read that slot, so the version is the only discriminator — hence opt-in. - * Clamped to >= 0.4.4. Returns false when unset or unparsable. - */ -bool -interactive_min_from_env(uint64_t *out) -{ - const char *e = std::getenv("DXR_LEIA_LIFT_INTERACTIVE_MIN"); - if (e == nullptr || e[0] == '\0') { - return false; - } - // Strict "MAJ.MIN.PAT" (no sscanf: MSVC C4996). - unsigned long part[3] = {0, 0, 0}; - const char *c = e; - bool ok = true; - for (int i = 0; i < 3 && ok; i++) { - char *end = nullptr; - part[i] = std::strtoul(c, &end, 10); - ok = end != c && *end == (i < 2 ? '.' : '\0'); - c = end + (i < 2 ? 1 : 0); - } - if (!ok) { - U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN='%s' not a version (e.g. 0.4.4) — ignored", e); - return false; - } - const uint64_t floor_v = NEURD_MAKE_VERSION(0, 4, 4); - const uint64_t v = NEURD_MAKE_VERSION(part[0], part[1], part[2]); - *out = (v < floor_v) ? floor_v : v; - return true; -} - /*! * NeurD_convert_stream_dx_interactive, honouring DXR_LEIA_LIFT_INTERACTIVE_MIN: * the header's inline wrapper re-checks version >= INTRODUCED_IN (0.4.5), so @@ -791,14 +922,27 @@ activation_worker() std::lock_guard lock(g.mtx); g.backend = be; snprintf(g.backend_name, sizeof(g.backend_name), "neurd-%s", backend_str(be)); + const struct knobs &k0 = g.k0; uint64_t interactive_min = NEURD_convert_stream_dx_interactive_INTRODUCED_IN; - if (interactive_min_from_env(&interactive_min)) { - U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN=%u.%u.%u — assuming interactive convert on NeurD " + if (k0.interactive_min != 0) { + interactive_min = k0.interactive_min; + } + U_LOG_W("Leia lift: knobs backend=%s(%s) interactive_min=%u.%u.%u(%s) scale=%s(%s) view_gain=%.2f(%s) " + "conv_gain=%.2f(%s) [env > HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift > default]", + backend_choice_str(k0.backend), knob_src_str(k0.src_backend), + (unsigned)NEURD_GET_VERSION_MAJOR(interactive_min), (unsigned)NEURD_GET_VERSION_MINOR(interactive_min), + (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), knob_src_str(k0.src_interactive_min), + k0.scale_forced ? scale_str(k0.autoscaling) : "per-stream", knob_src_str(k0.src_scale), + (double)k0.view_gain, knob_src_str(k0.src_view_gain), (double)k0.conv_gain, + knob_src_str(k0.src_conv_gain)); + if (k0.interactive_min != 0) { + U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN=%u.%u.%u (%s) — assuming interactive convert on NeurD " "%u.%u.%u (only for the internal-interactive dev package; a stock 0.4.4 will crash here)", (unsigned)NEURD_GET_VERSION_MAJOR(interactive_min), (unsigned)NEURD_GET_VERSION_MINOR(interactive_min), - (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), (unsigned)NEURD_GET_VERSION_MAJOR(nd->version), - (unsigned)NEURD_GET_VERSION_MINOR(nd->version), (unsigned)NEURD_GET_VERSION_PATCH(nd->version)); + (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), knob_src_str(k0.src_interactive_min), + (unsigned)NEURD_GET_VERSION_MAJOR(nd->version), (unsigned)NEURD_GET_VERSION_MINOR(nd->version), + (unsigned)NEURD_GET_VERSION_PATCH(nd->version)); } g.interactive_unavailable = !(nd->version >= interactive_min && nd->convert_stream_dx_interactive != nullptr); g.interactive_direct_slot = !g.interactive_unavailable.load() && @@ -819,7 +963,7 @@ activation_worker() * touch, and (re)start the activation worker when due. */ void -kick(int requested_backend, int32_t default_autoscaling) +kick(const struct knobs &k) { uint32_t s = g.state.load(); if (s == G_UNPROBED) { @@ -846,8 +990,9 @@ kick(int requested_backend, int32_t default_autoscaling) return; } if (s == G_UNPROBED) { - g.requested_backend = requested_backend; - g.default_autoscaling = default_autoscaling; + g.requested_backend = k.backend; + g.default_autoscaling = k.autoscaling; + g.k0 = k; } g.state.store(G_LOADING); try { @@ -1312,7 +1457,7 @@ leia_lift_neurd_get_caps(struct leia_lift_neurd *l, struct leia_lift_neurd_caps if (l == nullptr || !l->k.enabled) { return true; // modes 0 / UNAVAILABLE — a valid answer, not a failure. } - kick(l->k.backend, l->k.autoscaling); + kick(l->k); uint32_t s = g.state.load(); out->state = public_state(s); @@ -1364,7 +1509,7 @@ leia_lift_neurd_stream_create(struct leia_lift_neurd *l, const struct leia_lift_ LIFT_WARN_ONCE("Leia lift: stream_create with unsupported mode %u", desc->mode); return false; } - kick(l->k.backend, l->k.autoscaling); + kick(l->k); if (public_state(g.state.load()) == LEIA_LIFT_STATE_UNAVAILABLE) { return false; } @@ -1436,7 +1581,7 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, } *out_resource = nullptr; - kick(l->k.backend, l->k.autoscaling); + kick(l->k); if (g.state.load() != G_READY) { return false; // ACTIVATING or UNAVAILABLE — caps says which. } From e036d3f0401797ac1547d75b120b4d5a36a61cb2 Mon Sep 17 00:00:00 2001 From: dfattal Date: Sat, 26 Sep 2026 18:32:52 -0700 Subject: [PATCH 16/21] feat(lift): VideoModel knob -- pick NeurD 0.4.6's metric video model DXR_LEIA_LIFT_VIDEO_MODEL / HKLM\SOFTWARE\DisplayXR\Leia\Lift VideoModel: 'fast' (default, NEURD_MODEL_VIDEO_RELATIVE_FAST, unchanged) or 'metric' (NEURD_MODEL_VIDEO_METRIC_QUALITY) for NeurD_init_with_options' video slot, so the metric real-time model can be A/B'd for fps and quality without a rebuild. Photo streams keep the relative photo model (the only one that slot accepts). The activation knobs WARN reports it with its source. Co-Authored-By: Claude Opus 5.5 --- src/drv_leia/leia_lift_neurd.cpp | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index e38dbc4..afea67a 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -215,7 +215,8 @@ struct knobs float view_gain; //!< DXR_LEIA_LIFT_VIEW_GAIN / ViewGain (default 1.0) float conv_gain; //!< DXR_LEIA_LIFT_CONV_GAIN / ConvGain (default 0.4): relative convergence -> NeurD units uint64_t interactive_min; //!< DXR_LEIA_LIFT_INTERACTIVE_MIN / InteractiveMin; 0 = unset (header's INTRODUCED_IN) - knob_src src_backend, src_scale, src_view_gain, src_conv_gain, src_interactive_min; + int32_t video_model; //!< DXR_LEIA_LIFT_VIDEO_MODEL / VideoModel: fast (default) | metric (NeurD >= 0.4.6) + knob_src src_backend, src_scale, src_view_gain, src_conv_gain, src_interactive_min, src_video_model; }; bool @@ -364,6 +365,7 @@ read_knobs() k.view_gain = 1.0f; k.conv_gain = 0.4f; k.interactive_min = 0; + k.video_model = NEURD_MODEL_VIDEO_RELATIVE_FAST; const char *e = std::getenv("DXR_LEIA_LIFT"); if (e != nullptr && (e[0] == '0' || env_ieq(e, "off") || env_ieq(e, "false"))) { @@ -429,6 +431,24 @@ read_knobs() } } + // The video streams' depth model (NeurD_init_with_options, 0.4.6+): 'fast' = the relative + // real-time model, 'metric' = NEURD_MODEL_VIDEO_METRIC_QUALITY (metric depth, closer to what the + // web path's MoGe gives). Photo streams always use the relative photo model (the only one the + // photo slot accepts). + src = knob_lookup(reg, "DXR_LEIA_LIFT_VIDEO_MODEL", L"VideoModel", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (env_ieq(v, "metric")) { + k.video_model = NEURD_MODEL_VIDEO_METRIC_QUALITY; + k.src_video_model = src; + } else if (env_ieq(v, "fast")) { + k.video_model = NEURD_MODEL_VIDEO_RELATIVE_FAST; + k.src_video_model = src; + } else { + U_LOG_W("Leia lift: video model '%s' (%s) not recognised (fast|metric) — using fast", v, + knob_src_str(src)); + } + } + if (reg != nullptr) { RegCloseKey(reg); } @@ -888,7 +908,7 @@ activation_worker() struct NeurD_init_options opts = {}; opts.struct_size = sizeof(opts); opts.photo_model_id = NEURD_MODEL_PHOTO_RELATIVE_QUALITY; - opts.video_model_id = NEURD_MODEL_VIDEO_RELATIVE_FAST; + opts.video_model_id = g.k0.video_model; st = NeurD_init_with_options(nd, 1 /* multithreaded: we call from the runtime's lift thread */, &opts); } else if (LEIA_NEURD_HAS(nd, init)) { st = NeurD_init(nd, 1); @@ -928,13 +948,15 @@ activation_worker() interactive_min = k0.interactive_min; } U_LOG_W("Leia lift: knobs backend=%s(%s) interactive_min=%u.%u.%u(%s) scale=%s(%s) view_gain=%.2f(%s) " - "conv_gain=%.2f(%s) [env > HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift > default]", + "conv_gain=%.2f(%s) video_model=%s(%s) [env > HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift > default]", backend_choice_str(k0.backend), knob_src_str(k0.src_backend), (unsigned)NEURD_GET_VERSION_MAJOR(interactive_min), (unsigned)NEURD_GET_VERSION_MINOR(interactive_min), (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), knob_src_str(k0.src_interactive_min), k0.scale_forced ? scale_str(k0.autoscaling) : "per-stream", knob_src_str(k0.src_scale), (double)k0.view_gain, knob_src_str(k0.src_view_gain), (double)k0.conv_gain, - knob_src_str(k0.src_conv_gain)); + knob_src_str(k0.src_conv_gain), + k0.video_model == NEURD_MODEL_VIDEO_METRIC_QUALITY ? "metric" : "fast", + knob_src_str(k0.src_video_model)); if (k0.interactive_min != 0) { U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN=%u.%u.%u (%s) — assuming interactive convert on NeurD " "%u.%u.%u (only for the internal-interactive dev package; a stock 0.4.4 will crash here)", From 8a73cd95ef963fa76210c8380b663f13124f0728 Mon Sep 17 00:00:00 2001 From: dfattal Date: Sat, 26 Sep 2026 18:41:38 -0700 Subject: [PATCH 17/21] feat(lift): log which viewpoints a convert used (tracked / explicit / default) SBS only sends viewpoints when the eyes are tracked (or given explicitly); otherwise NeurD renders its default pattern and DXR_LEIA_LIFT_VIEW_GAIN is never applied. A ViewGain A/B run with nobody in front of the tracker therefore produced byte-identical output on NeurD 0.4.6 and read as 'the interactive path ignores our viewpoints'. One WARN per change of source makes the tell explicit. Co-Authored-By: Claude Opus 5.5 --- src/drv_leia/leia_lift_neurd.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index afea67a..b242ee4 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -1824,6 +1824,17 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, } } // SBS untracked: vp stays empty -> NeurD's own default ±0.5 pattern. + + // Which viewpoints this convert used, logged once per change: an A/B of + // DXR_LEIA_LIFT_VIEW_GAIN with nobody in front of the tracker lands on the + // default pattern and reads as "the gain does nothing". + const int vp_src = !vp.empty() ? (eyes_valid ? 1 : 2) : 0; + static std::atomic s_last_vp_src{-1}; + if (s_last_vp_src.exchange(vp_src) != vp_src) { + U_LOG_W("Leia lift: viewpoints = %s (mode %d, %u view(s)%s)", + vp_src == 1 ? "TRACKED eyes" : (vp_src == 2 ? "explicit/untracked N-view" : "DEFAULT pattern (no tracked eyes)"), + (int)s->mode, views, vp_src != 0 ? ", view_gain applied" : ", view_gain NOT applied"); + } } // ---- 5. Convert (blocking). From cda7f99999fb36d3bc3ced859d58d5d37df3cdc9 Mon Sep 17 00:00:00 2001 From: dfattal Date: Sat, 26 Sep 2026 22:59:37 -0700 Subject: [PATCH 18/21] feat(lift): DepthGain knob, default 2.0 -- NeurD gain = strength x DepthGain NeurD's GAIN_MULTIPLIER 1 read as too weak on the Leia panel; strength 2 (set live via dxr-lift-strength) was judged right as the default (David, 2026-09-26). The element's strength keeps meaning 1 = nominal, and the calibration lives here with NeurD's units: DXR_LEIA_LIFT_DEPTH_GAIN / HKLM\SOFTWARE\DisplayXR\Leia\Lift DepthGain (default 2.0), reported in the activation knobs WARN. (ViewGain is not a depth control: NeurD bounds the viewpoint excursion by its own disparity budget, so gain 2 = gain 1.) Co-Authored-By: Claude Opus 5.5 --- src/drv_leia/leia_lift_neurd.cpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index b242ee4..7c9b2f2 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -101,6 +101,8 @@ constexpr float kIpdRefM = 0.063f; //! Clamp for mapped viewpoints — well past any comfortable look-around; stops a //! tracker glitch from asking NeurD for a wildly extrapolated view. constexpr float kViewpointClamp = 3.0f; +//! NEURD_PROP_GAIN_MULTIPLIER at strength 1 (DepthGain knob). Panel-calibrated 2026-09-26. +constexpr float kDefaultDepthGain = 2.0f; //! NeurD's MAX_STREAMS. constexpr uint32_t kMaxStreams = 32; //! N views go side by side in ONE row (runtime contract), so the output is @@ -216,7 +218,8 @@ struct knobs float conv_gain; //!< DXR_LEIA_LIFT_CONV_GAIN / ConvGain (default 0.4): relative convergence -> NeurD units uint64_t interactive_min; //!< DXR_LEIA_LIFT_INTERACTIVE_MIN / InteractiveMin; 0 = unset (header's INTRODUCED_IN) int32_t video_model; //!< DXR_LEIA_LIFT_VIDEO_MODEL / VideoModel: fast (default) | metric (NeurD >= 0.4.6) - knob_src src_backend, src_scale, src_view_gain, src_conv_gain, src_interactive_min, src_video_model; + float depth_gain; //!< DXR_LEIA_LIFT_DEPTH_GAIN / DepthGain (default 2.0): NeurD gain at strength 1 + knob_src src_backend, src_scale, src_view_gain, src_conv_gain, src_interactive_min, src_video_model, src_depth_gain; }; bool @@ -366,6 +369,7 @@ read_knobs() k.conv_gain = 0.4f; k.interactive_min = 0; k.video_model = NEURD_MODEL_VIDEO_RELATIVE_FAST; + k.depth_gain = kDefaultDepthGain; const char *e = std::getenv("DXR_LEIA_LIFT"); if (e != nullptr && (e[0] == '0' || env_ieq(e, "off") || env_ieq(e, "false"))) { @@ -435,6 +439,16 @@ read_knobs() // real-time model, 'metric' = NEURD_MODEL_VIDEO_METRIC_QUALITY (metric depth, closer to what the // web path's MoGe gives). Photo streams always use the relative photo model (the only one the // photo slot accepts). + src = knob_lookup(reg, "DXR_LEIA_LIFT_DEPTH_GAIN", L"DepthGain", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + if (parse_float_in(v, 0.0f, 10.0f, &k.depth_gain)) { + k.src_depth_gain = src; + } else { + U_LOG_W("Leia lift: depth gain '%s' (%s) not a number in [0,10] — using %.1f", v, knob_src_str(src), + (double)kDefaultDepthGain); + } + } + src = knob_lookup(reg, "DXR_LEIA_LIFT_VIDEO_MODEL", L"VideoModel", v, sizeof(v)); if (src != KSRC_DEFAULT) { if (env_ieq(v, "metric")) { @@ -948,7 +962,7 @@ activation_worker() interactive_min = k0.interactive_min; } U_LOG_W("Leia lift: knobs backend=%s(%s) interactive_min=%u.%u.%u(%s) scale=%s(%s) view_gain=%.2f(%s) " - "conv_gain=%.2f(%s) video_model=%s(%s) [env > HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift > default]", + "conv_gain=%.2f(%s) video_model=%s(%s) depth_gain=%.2f(%s) [env > HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift > default]", backend_choice_str(k0.backend), knob_src_str(k0.src_backend), (unsigned)NEURD_GET_VERSION_MAJOR(interactive_min), (unsigned)NEURD_GET_VERSION_MINOR(interactive_min), (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), knob_src_str(k0.src_interactive_min), @@ -956,7 +970,7 @@ activation_worker() (double)k0.view_gain, knob_src_str(k0.src_view_gain), (double)k0.conv_gain, knob_src_str(k0.src_conv_gain), k0.video_model == NEURD_MODEL_VIDEO_METRIC_QUALITY ? "metric" : "fast", - knob_src_str(k0.src_video_model)); + knob_src_str(k0.src_video_model), (double)k0.depth_gain, knob_src_str(k0.src_depth_gain)); if (k0.interactive_min != 0) { U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN=%u.%u.%u (%s) — assuming interactive convert on NeurD " "%u.%u.%u (only for the internal-interactive dev package; a stock 0.4.4 will crash here)", @@ -1744,7 +1758,11 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, } const bool auto_conv = !(dp.convergence >= 0.0f); // strength: 1 = NeurD's calibrated budget, 0 = flat; negative/NaN = default. - const float gain = (dp.strength >= 0.0f) ? clampf(dp.strength, 0.0f, 10.0f) : 1.0f; + // The element's strength (1 = nominal) times the depth-gain calibration: + // NeurD's own gain 1 read as too weak on the panel (David, 2026-09-26: + // strength 2 "stronger, I like this as default"). + const float strength = (dp.strength >= 0.0f) ? dp.strength : 1.0f; + const float gain = clampf(strength * l->k.depth_gain, 0.0f, 10.0f); // NeurD always fills disocclusions; non-zero picks the blur fill, 0 the // cheaper edge stretch. const int32_t inpaint = (dp.inpaint != 0) ? NEURD_INPAINT_TYPE_V1_BLUR : NEURD_INPAINT_TYPE_V1_STRETCH; From 874a55348a2173ae7a475358afb06df98a4002fd Mon Sep 17 00:00:00 2001 From: dfattal Date: Sat, 26 Sep 2026 22:59:38 -0700 Subject: [PATCH 19/21] fix(lift): never pass the tracked eye HEIGHT to NeurD (vertical parallax) leia_lift_neurd_map_viewpoint mapped the eye y offset into NeurD's viewpoint, so with the viewer above the display centre NeurD rendered the lifted frame as seen from above: the image shifted vertically with head height ('the converted asset jumps up/down') and a mirror-filled band appeared at the top edge. Weave dumps on the panel: both eyes shifted +7 px (top) / -7 px (bottom) vs the element, where the untracked probe shows NeurD's own +/-4 px. A lenticular panel has horizontal parallax only, so y is zeroed. (x keeps the eye midpoint = horizontal look-around.) Co-Authored-By: Claude Opus 5.5 --- src/drv_leia/leia_lift_neurd.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index 7c9b2f2..c82f65c 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -1433,7 +1433,13 @@ extern "C" void leia_lift_neurd_map_viewpoint(const float in_m[3], float gain, float out_n[3]) { out_n[0] = clampf(gain * in_m[0] / kIpdRefM, -kViewpointClamp, kViewpointClamp); - out_n[1] = clampf(gain * in_m[1] / kIpdRefM, -kViewpointClamp, kViewpointClamp); + // A lenticular panel has horizontal parallax only. Passing the tracked eye + // HEIGHT made NeurD render the frame as seen from above/below the display + // centre: the lifted image shifted vertically with head height (visible + // "jumps" as the head moved) and a filled band appeared at the top edge + // (David, panel, 2026-09-26). Vertical viewpoint offset is never wanted. + (void)in_m[1]; + out_n[1] = 0.0f; // NeurD's z is a dimensionless depth offset whose relation to viewer // distance is not specified; head z is deliberately not mapped (see doc). out_n[2] = 0.0f; From ad147ee14a468d774d7b6bda86384bb421a5a37f Mon Sep 17 00:00:00 2001 From: dfattal Date: Sat, 26 Sep 2026 23:19:35 -0700 Subject: [PATCH 20/21] feat(lift): Dilate knob, default 2 (NeurD's own default is 3) NEURD_PROP_DILATE_RADIO was never set, so NeurD dilated the disparity map by its default 3 px and the foreground visibly grew past its silhouette on the panel (David, 2026-09-26: 'the disparity map dilation is a bit too much, reduce a bit'). Now set every convert (cached like the other props) from DXR_LEIA_LIFT_DILATE / HKLM\SOFTWARE\DisplayXR\Leia\Lift Dilate, default 2; reported in the activation knobs WARN. Co-Authored-By: Claude Opus 5.5 --- src/drv_leia/leia_lift_neurd.cpp | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/drv_leia/leia_lift_neurd.cpp b/src/drv_leia/leia_lift_neurd.cpp index c82f65c..9a45340 100644 --- a/src/drv_leia/leia_lift_neurd.cpp +++ b/src/drv_leia/leia_lift_neurd.cpp @@ -103,6 +103,10 @@ constexpr float kIpdRefM = 0.063f; constexpr float kViewpointClamp = 3.0f; //! NEURD_PROP_GAIN_MULTIPLIER at strength 1 (DepthGain knob). Panel-calibrated 2026-09-26. constexpr float kDefaultDepthGain = 2.0f; +//! NEURD_PROP_DILATE_RADIO (Dilate knob). NeurD's own default is 3; the foreground grew visibly +//! past its silhouette on the panel (David, 2026-09-26: "the disparity map dilation is a bit too +//! much, reduce a bit"). +constexpr int32_t kDefaultDilate = 2; //! NeurD's MAX_STREAMS. constexpr uint32_t kMaxStreams = 32; //! N views go side by side in ONE row (runtime contract), so the output is @@ -219,7 +223,9 @@ struct knobs uint64_t interactive_min; //!< DXR_LEIA_LIFT_INTERACTIVE_MIN / InteractiveMin; 0 = unset (header's INTRODUCED_IN) int32_t video_model; //!< DXR_LEIA_LIFT_VIDEO_MODEL / VideoModel: fast (default) | metric (NeurD >= 0.4.6) float depth_gain; //!< DXR_LEIA_LIFT_DEPTH_GAIN / DepthGain (default 2.0): NeurD gain at strength 1 - knob_src src_backend, src_scale, src_view_gain, src_conv_gain, src_interactive_min, src_video_model, src_depth_gain; + int32_t dilate; //!< DXR_LEIA_LIFT_DILATE / Dilate (default 2): NEURD_PROP_DILATE_RADIO, px + knob_src src_backend, src_scale, src_view_gain, src_conv_gain, src_interactive_min, src_video_model, src_depth_gain, + src_dilate; }; bool @@ -370,6 +376,7 @@ read_knobs() k.interactive_min = 0; k.video_model = NEURD_MODEL_VIDEO_RELATIVE_FAST; k.depth_gain = kDefaultDepthGain; + k.dilate = kDefaultDilate; const char *e = std::getenv("DXR_LEIA_LIFT"); if (e != nullptr && (e[0] == '0' || env_ieq(e, "off") || env_ieq(e, "false"))) { @@ -449,6 +456,18 @@ read_knobs() } } + src = knob_lookup(reg, "DXR_LEIA_LIFT_DILATE", L"Dilate", v, sizeof(v)); + if (src != KSRC_DEFAULT) { + float d = 0.0f; + if (parse_float_in(v, 0.0f, 16.0f, &d)) { + k.dilate = (int32_t)(d + 0.5f); + k.src_dilate = src; + } else { + U_LOG_W("Leia lift: dilate '%s' (%s) not a number in [0,16] — using %d", v, knob_src_str(src), + (int)kDefaultDilate); + } + } + src = knob_lookup(reg, "DXR_LEIA_LIFT_VIDEO_MODEL", L"VideoModel", v, sizeof(v)); if (src != KSRC_DEFAULT) { if (env_ieq(v, "metric")) { @@ -579,6 +598,7 @@ struct global bool props_valid = false; int32_t p_out_type = -1, p_tiles_w = -1, p_tiles_h = -1, p_inpaint = -1, p_autoscale = -1, p_autoconv = -1; float p_conv = -1.0f, p_gain = -1.0f; + int32_t p_dilate = -1; }; global g; @@ -962,7 +982,7 @@ activation_worker() interactive_min = k0.interactive_min; } U_LOG_W("Leia lift: knobs backend=%s(%s) interactive_min=%u.%u.%u(%s) scale=%s(%s) view_gain=%.2f(%s) " - "conv_gain=%.2f(%s) video_model=%s(%s) depth_gain=%.2f(%s) [env > HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift > default]", + "conv_gain=%.2f(%s) video_model=%s(%s) depth_gain=%.2f(%s) dilate=%d(%s) [env > HKLM\\SOFTWARE\\DisplayXR\\Leia\\Lift > default]", backend_choice_str(k0.backend), knob_src_str(k0.src_backend), (unsigned)NEURD_GET_VERSION_MAJOR(interactive_min), (unsigned)NEURD_GET_VERSION_MINOR(interactive_min), (unsigned)NEURD_GET_VERSION_PATCH(interactive_min), knob_src_str(k0.src_interactive_min), @@ -970,7 +990,8 @@ activation_worker() (double)k0.view_gain, knob_src_str(k0.src_view_gain), (double)k0.conv_gain, knob_src_str(k0.src_conv_gain), k0.video_model == NEURD_MODEL_VIDEO_METRIC_QUALITY ? "metric" : "fast", - knob_src_str(k0.src_video_model), (double)k0.depth_gain, knob_src_str(k0.src_depth_gain)); + knob_src_str(k0.src_video_model), (double)k0.depth_gain, knob_src_str(k0.src_depth_gain), (int)k0.dilate, + knob_src_str(k0.src_dilate)); if (k0.interactive_min != 0) { U_LOG_W("Leia lift: DXR_LEIA_LIFT_INTERACTIVE_MIN=%u.%u.%u (%s) — assuming interactive convert on NeurD " "%u.%u.%u (only for the internal-interactive dev package; a stock 0.4.4 will crash here)", @@ -1780,7 +1801,8 @@ leia_lift_neurd_convert(struct leia_lift_neurd *l, prop_i(NEURD_PROP_INPUT_AUTOSCALING, autoscale_for(s, l->k, h), g.p_autoscale) && prop_i(NEURD_PROP_AUTO_CONVERGENCE, auto_conv ? 1 : 0, g.p_autoconv) && - prop_f(NEURD_PROP_GAIN_MULTIPLIER, gain, g.p_gain); + prop_f(NEURD_PROP_GAIN_MULTIPLIER, gain, g.p_gain) && + prop_i(NEURD_PROP_DILATE_RADIO, l->k.dilate, g.p_dilate); if (props_ok && !auto_conv) { // Runtime: convergence = RELATIVE depth placed at the display plane, // [0,1] over the frame's depth range (0 = nearest on the glass, 1 = From f86015d534d7ff97fb7c64815badf38f01b4073d Mon Sep 17 00:00:00 2001 From: dfattal Date: Thu, 1 Oct 2026 14:25:28 -0700 Subject: [PATCH 21/21] =?UTF-8?q?build(windows):=20pin=20the=20runtime=20t?= =?UTF-8?q?o=20v2.22.0=20=E2=80=94=20first=20release=20with=20the=20D3D11?= =?UTF-8?q?=20lift=20slots?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NeurD lift module compiles only against runtime headers that define XRT_DP_D3D11_HAS_LIFT (runtime#1781, released in v2.22.0). The slots are appended and struct_size-gated, so the plug-in ABI stays v5; the installer's derived minimum-runtime floor moves to 2.22.0. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/build-windows.yml | 2 +- CMakeLists.txt | 6 +++++- docs/lift-neurd.md | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 9cb33c2..021c82d 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -109,7 +109,7 @@ jobs: # regression originally). # #734: runtime v2.0.0 (a8f6e74f) ships the XR_EXT_* → XR_DXR_* # extension rename this plug-in's headers now use (ABI unchanged, v4). - RUNTIME_REF: v2.16.9 + RUNTIME_REF: v2.22.0 steps: - name: Checkout uses: actions/checkout@v4 diff --git a/CMakeLists.txt b/CMakeLists.txt index ea8425c..a5d88d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -162,9 +162,13 @@ endif() # This plug-in MUST be built against headers that ship the matching major, # or the runtime will log "ABI major mismatch — plugin_api=N, runtime # expects M ... skipping (ADR-020 rule 3)" and fall back to sim_display. +# v2.22.0 (from v2.16.9): the first runtime that ships the D3D11 DP lift slots +# (XRT_DP_D3D11_HAS_LIFT, runtime#1781) and the lift-only factory +# (XRT_PLUGIN_IFACE_HAS_D3D11_LIFT_FACTORY). Appended and struct_size-gated, +# plug-in ABI still v5; an older pin compiles the NeurD lift module OUT. # The rule-5 CI self-check below asserts this value equals the workflow's # runtime checkout ref so the two can't drift. -set(DXR_RUNTIME_GIT_TAG "v2.16.9" CACHE STRING +set(DXR_RUNTIME_GIT_TAG "v2.22.0" CACHE STRING "Runtime git ref to fetch. Must equal the workflow's runtime checkout ref (rule-5 CI check enforces).") # Linux carries its own pin so it can move independently of the Windows # installer's derived MIN_RUNTIME_VERSION (same reason as _ANDROID below). diff --git a/docs/lift-neurd.md b/docs/lift-neurd.md index d4db23c..6eb80f1 100644 --- a/docs/lift-neurd.md +++ b/docs/lift-neurd.md @@ -301,12 +301,12 @@ ref; it is the table version the module requests from `NeurD_load`. ### 2. Runtime headers with the lift slots -The slots compile only when the runtime headers define `XRT_DP_D3D11_HAS_LIFT`. The -Windows pin `DXR_RUNTIME_GIT_TAG` is currently `v2.16.9`, which predates it — a pinned -build ships the plug-in with the lift slots compiled out (it logs +The slots compile only when the runtime headers define `XRT_DP_D3D11_HAS_LIFT`, which +first ships in runtime `v2.22.0` — the Windows pin `DXR_RUNTIME_GIT_TAG`. Built against +older headers the plug-in ships with the lift slots compiled out (it logs `lift slots NOT COMPILED`), and the runtime sees them absent via `struct_size`. -To build with lift before the runtime tags it, use a local runtime checkout — nothing +To build against an unreleased runtime, use a local runtime checkout — nothing is committed: ```bat