diff --git a/samples/desktop-avatar/Assets/TigerFaceViewer.cs b/samples/desktop-avatar/Assets/TigerFaceViewer.cs new file mode 100644 index 0000000..67ca6c5 --- /dev/null +++ b/samples/desktop-avatar/Assets/TigerFaceViewer.cs @@ -0,0 +1,261 @@ +// Copyright 2024-2026, DisplayXR contributors +// SPDX-License-Identifier: Apache-2.0 + +// Optional head-tracked billboard for the tiger (yaw-only) + A/B probe for the +// ways an app can ask "where is the viewer?" (plugin issue #236). +// +// The yaw math mirrors the native reference implementation in +// displayxr-demo-avatar (windows/main.cpp, "Face-the-viewer billboard") 1:1, +// because that one is viewer-confirmed on hardware: +// +// targetYaw = FACE_YAW_SIGN * atan2(hx, |hz|) +// +// with (hx, hz) = the head centroid in RAW DISPLAY SPACE — metres, origin at the +// physical panel centre, viewer at +Z — from displayxr_get_eye_positions. That is +// deliberately NOT the world-space render-ready eyes: physical space is +// independent of where the rig, camera and model sit in the scene, and immune to +// the display rig's scale-as-zoom (which inflates world-space distances, e.g. a +// 0.06 m IPD reads as ~1.2 world units here). Two earlier attempts to derive the +// angle in world space failed exactly there — one silently rotated the wrong +// object, the next found its reference vector degenerate because the tiger's root +// sits at the camera's x/z. +// +// Smoothing is the reference's time-based exponential (tau), and the heading only +// chases while eye tracking is LOCKED — warmup positions jitter the heading. +// +// The once-per-second log still prints the world-space provider API +// (DisplayXRProvider.TryGetViewerEyes, new in v2.9.1) next to the raw physical +// eyes and to Camera.GetStereoViewMatrix, so the sources stay comparable — that +// A/B is the point of the probe for issue #236. +// +// Auto-installs via RuntimeInitializeOnLoadMethod (the KooimaProbe pattern), so +// it needs no scene edit. Off by default — press F. +// +// Not ported from the reference: the zone-canvas rebase (it offsets hx by the +// window's centre on the panel for off-centre windows). Without it the heading is +// referenced to the PANEL centre rather than the window centre — a constant bias +// when the window is off-centre, not a tracking failure. + +using System; +using DisplayXR; +using UnityEngine; +using UnityEngine.InputSystem; + +[DisallowMultipleComponent] +public class TigerFaceViewer : MonoBehaviour +{ + [Tooltip("Toggles head-tracked billboarding on/off. On by default, so F " + + "turns it OFF and restores the tiger's authored rotation.")] + public Key toggleKey = Key.F; + + [Tooltip("Billboard from startup. The tiger's DragRotateCube is wired by " + + "TransparentAutoSetup at AfterSceneLoad, same phase as this " + + "component, so enabling retries until the target exists.")] + public bool startEnabled = true; + + [Tooltip("Yaw direction. -1 matches the viewer-confirmed native reference " + + "(displayxr-demo-avatar FACE_YAW_SIGN); flip to +1 if the tiger " + + "turns away from you instead of toward you.")] + public float yawSign = -1f; + + [Tooltip("Multiplies the viewer's off-axis angle. 1 = physically faithful " + + "(what the native reference does); >1 exaggerates for a demo.")] + public float gain = 1f; + + [Tooltip("Smoothing time constant in seconds (settle ~= 3x tau). Matches the " + + "reference's 0.04. Time-based, so it is frame-rate independent.")] + public float tau = 0.04f; + + [Tooltip("Clamp on the applied yaw (degrees).")] + public float maxYawDegrees = 60f; + + [Tooltip("Seconds between A/B probe log lines. 0 = don't log.")] + public float logInterval = 1f; + + private Transform m_Tiger; + private Quaternion m_BaseRotation; + private bool m_Enabled; + private float m_Timer; + private float m_Yaw; // smoothed, applied + private float m_TargetYaw; // pre-smoothing, for the log + private bool m_UserTurnedOff; // an explicit F press wins over startEnabled + private float m_RetryTimer; + + private static bool s_RawMissing; + + [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.AfterSceneLoad)] + static void AutoInstall() + { + if (FindAnyObjectByType() != null) return; + var go = new GameObject("TigerFaceViewer"); + DontDestroyOnLoad(go); + go.AddComponent(); + Debug.Log("[TigerFaceViewer] installed — head-tracked billboard ON by default (press F to turn it off)."); + } + + void Update() + { + var kb = Keyboard.current; + if (kb != null && toggleKey != Key.None && kb[toggleKey].wasPressedThisFrame) + { + m_UserTurnedOff = m_Enabled; // F while on = the user wants it off + Toggle(); + } + + // Auto-enable at startup. TransparentAutoSetup adds the tiger's + // DragRotateCube in the same AfterSceneLoad phase we install in, and the + // order between them is undefined — so retry rather than assume the + // target exists on frame one. Throttled: FindObjectsByType is not free. + if (startEnabled && !m_Enabled && !m_UserTurnedOff) + { + m_RetryTimer -= Time.deltaTime; + if (m_RetryTimer <= 0f) + { + m_RetryTimer = 0.25f; + if (FindTiger() != null) Toggle(); + } + } + + if (logInterval > 0f && m_Enabled) + { + m_Timer += Time.deltaTime; + if (m_Timer >= logInterval) { m_Timer = 0f; LogSources(); } + } + } + + void Toggle() + { + if (m_Enabled) + { + m_Enabled = false; + if (m_Tiger != null) m_Tiger.rotation = m_BaseRotation; + Debug.Log("[TigerFaceViewer] OFF (rotation restored)"); + return; + } + + m_Tiger = FindTiger(); + if (m_Tiger == null) + { + Debug.LogWarning("[TigerFaceViewer] no DragRotateCube target found — staying off."); + return; + } + + m_BaseRotation = m_Tiger.rotation; + m_Yaw = 0f; + m_Enabled = true; + m_Timer = logInterval; // log immediately + + var rend = m_Tiger.GetComponentInChildren(true); + bool raw = TryGetRawHead(out float hx, out float hy, out float hz, out bool tracked); + Debug.Log($"[TigerFaceViewer] ON target={m_Tiger.name} " + + $"renderer={(rend != null ? rend.GetType().Name + "/" + rend.name : "")} " + + $"pos={Fmt(m_Tiger.position)} yawSign={yawSign} gain={gain:F1} " + + $"| rawHead ok={raw} ({hx:F3},{hy:F3},{hz:F3}) tracked={tracked}"); + } + + void LateUpdate() + { + // LateUpdate so we win over the Animator (which runs earlier in the frame) + // and over DragRotateCube's drag rotation while this mode is on. + if (!m_Enabled || m_Tiger == null) return; + if (!TryGetRawHead(out float hx, out float _, out float hz, out bool tracked)) return; + + // Reference math, 1:1: yaw from the head's lateral offset over its + // distance to the panel. |hz| guarded so a viewer in the panel plane + // can't blow the angle up. + float hzAbs = Mathf.Max(Mathf.Abs(hz), 1e-3f); + m_TargetYaw = Mathf.Clamp(yawSign * Mathf.Atan2(hx, hzAbs) * Mathf.Rad2Deg * gain, + -maxYawDegrees, maxYawDegrees); + + // Time-based exponential smoothing (frame-rate independent). Only chase + // while tracking is locked — warmup positions jitter the heading. + if (tracked) + { + float a = Mathf.Clamp01(1f - Mathf.Exp(-Time.deltaTime / Mathf.Max(tau, 1e-4f))); + m_Yaw = Mathf.LerpAngle(m_Yaw, m_TargetYaw, a); + } + + m_Tiger.rotation = Quaternion.AngleAxis(m_Yaw, Vector3.up) * m_BaseRotation; + } + + // Print every viewer-position source together: the raw physical eyes that + // drive the billboard, the world-space provider API (new in v2.9.1), and the + // Camera.GetStereoViewMatrix readback that issue #236 was filed about. + void LogSources() + { + bool raw = TryGetRawHead(out float hx, out float hy, out float hz, out bool tracked); + + bool okProv = DisplayXRProvider.TryGetViewerEyes(out Vector3 pl, out Vector3 pr); + Vector3 pHead = okProv ? (pl + pr) * 0.5f : Vector3.zero; + + var cam = ActiveCamera(); + bool stereo = cam != null && cam.stereoEnabled; + Vector3 sl = Vector3.zero, sr = Vector3.zero; + if (stereo) + { + sl = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Left).inverse.GetColumn(3); + sr = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Right).inverse.GetColumn(3); + } + bool camEyesEqual = stereo && (sl - sr).sqrMagnitude < 1e-10f; + + Debug.Log($"[TigerFaceViewer] rawHead ok={raw} ({hx:F3},{hy:F3},{hz:F3})m tracked={tracked} " + + $"-> targetYaw={m_TargetYaw:F1} yaw={m_Yaw:F1}deg on {(m_Tiger != null ? m_Tiger.name : "")} " + + $"| provider(world) ok={okProv} head={Fmt(pHead)} ipd={(pr - pl).magnitude:F4} " + + $"| GetStereoViewMatrix stereo={stereo} eyesEqual={camEyesEqual} " + + $"| eyeTracked={DisplayXRProvider.IsEyeTracked}"); + } + + // Head centroid in RAW DISPLAY SPACE: metres, origin at the physical panel + // centre, +X right, +Y up, viewer at +Z. Same channel the Scene-view eye + // gizmo uses (DisplayXRGizmoHelpers.TryGetLiveRawEyes) and the same one the + // native reference billboards from. + static bool TryGetRawHead(out float hx, out float hy, out float hz, out bool tracked) + { + hx = 0f; hy = 0f; hz = 0f; tracked = false; + if (s_RawMissing) return false; + try + { + DisplayXRNative.displayxr_get_eye_positions( + out float lx, out float ly, out float lz, + out float rx, out float ry, out float rz, + out int isTracked); + hx = (lx + rx) * 0.5f; + hy = (ly + ry) * 0.5f; + hz = (lz + rz) * 0.5f; + tracked = isTracked != 0; + // All-zero = the runtime hasn't filled the raw channel yet. + if (hx == 0f && hy == 0f && hz == 0f) return false; + return true; + } + catch (DllNotFoundException) { s_RawMissing = true; return false; } + catch (EntryPointNotFoundException) { s_RawMissing = true; return false; } + } + + static string Fmt(Vector3 v) => $"({v.x:F3},{v.y:F3},{v.z:F3})"; + + static Camera ActiveCamera() + { + var cam = DisplayXRRigManager.ActiveCamera; + return cam != null ? cam : Camera.main; + } + + // DragRotateCube owns each target's drag rotation, so it is the handle we + // want — but TransparentAutoSetup wires one onto EVERY target root, which is + // the tiger AND the legacy fallback cube. FindAnyObjectByType returns an + // arbitrary one, and picking the cube silently rotates an invisible fixture + // while the tiger sits still (looks exactly like "head tracking is dead"). + // Prefer the skinned mesh — that's the tiger, and it needs no name match. + static Transform FindTiger() + { + var drags = FindObjectsByType(); + if (drags == null || drags.Length == 0) return null; + foreach (var d in drags) + if (d.target is SkinnedMeshRenderer) return d.transform; + // No skinned mesh (cube-only fallback scene): take the first target that + // is actually being drawn. + foreach (var d in drags) + if (d.target != null && d.target.enabled && d.target.gameObject.activeInHierarchy) + return d.transform; + return drags[0].transform; + } +} diff --git a/samples/desktop-avatar/Assets/TigerFaceViewer.cs.meta b/samples/desktop-avatar/Assets/TigerFaceViewer.cs.meta new file mode 100644 index 0000000..9dd454c --- /dev/null +++ b/samples/desktop-avatar/Assets/TigerFaceViewer.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: f273ffa5104345db84dc4041ac545f77 \ No newline at end of file diff --git a/samples/desktop-avatar/CLAUDE.md b/samples/desktop-avatar/CLAUDE.md index bcdb199..1a1e00d 100644 --- a/samples/desktop-avatar/CLAUDE.md +++ b/samples/desktop-avatar/CLAUDE.md @@ -44,6 +44,7 @@ All test-project components are runtime-wired by `TransparentAutoSetup` — ther | `WheelZoomVHeight` | `Assets/TransparentAutoSetup.cs` (nested) | Scroll-wheel → `DisplayXRDisplay.virtualDisplayHeight`. Display-centric only. Active-rig gated (only the focused rig drains the wheel accumulator). | | `LockToForwardAxis` | `Assets/LockToForwardAxis.cs` | **Tiger-branch tweak.** Locks the rig camera's world X/Y to its startup values each `Update`, after the plugin's `DisplayXRInputController` has moved it. Net effect: AQDE keys become no-ops, only W/S still push the camera in/out (so only the in/out-of-display-plane axis is user-controllable). Uses `[DefaultExecutionOrder(int.MaxValue)]` to run after the plugin's input controller. | | `ClipAtDisplayPlane` | `Assets/TransparentAutoSetup.cs` (nested) | **Work in progress.** Currently hooks `Camera.onPreCull` and rewrites the per-eye stereo projection's `m22`/`m23` (near/far elements) to clip at raw eye-Z. Tiger renders but the clip doesn't visibly take effect. See `docs~/handoff-foreground-clipping.md` and issue #2. | +| `TigerFaceViewer` | `Assets/TigerFaceViewer.cs` | **Head-tracked billboard (yaw-only), ON by default** — the tiger turns to face the tracked viewer; press **F** to turn it off and restore its authored rotation. Yaw math is ported 1:1 from the native reference (see below). The once-per-second log prints every viewer-position source side by side, which is also the A/B probe for plugin #236. Self-installs via `[RuntimeInitializeOnLoadMethod]` — no scene wiring. | | `AutoBoxColliderFromRenderer` | `Assets/TransparentAutoSetup.cs` (nested) | Deferred BoxCollider sizing for **non-SMR** clickables (the cube fallback). Waits for `renderer.bounds` to be valid (a few frames), then sizes the box. Skipped for `SkinnedMeshRenderer` — the plugin handles those per-triangle. | ## How transparency + clickthrough work @@ -56,6 +57,58 @@ The mechanism splits across plugin (most of it) and test-project (small bootstra 4. **Per-pixel click-through** — `WM_NCHITTEST` in the native overlay reads `s_hit_active` (set by the C# polling code each frame). When the cyclopean ray hits the tiger silhouette → `s_hit_active=1` → `HTCLIENT` → overlay captures. Otherwise `HTTRANSPARENT` → click forwards to the underlying app via `forward_click_to_underlying_window` (`SetForegroundWindow` + `PostMessage`). 5. **Hit testing** uses **per-triangle ray-tri** (Möller-Trumbore) against `SkinnedMeshRenderer.BakeMesh()` output, transformed via `Matrix4x4.TRS(smr.position, smr.rotation, Vector3.one)` — position + rotation only, no scale (BakeMesh's output is already in world units). 8-frame hysteresis smooths over silhouette-edge sub-pixel jitter. Active-rig gate prevents the two rigs from flapping `s_hit_active`. Implemented entirely in the plugin; the test project just sets `clickableRenderers` to the SMR. +## Head-coupled effects: work in PHYSICAL space, not world space + +**Billboard yaw is computed in raw display space, not Unity world space.** This is +the single most important lesson from building `TigerFaceViewer`, and it mirrors the +viewer-confirmed native reference in +[`displayxr-demo-avatar`](https://github.com/DisplayXR/displayxr-demo-avatar) +(`windows/main.cpp`, "Face-the-viewer billboard"): + +```csharp +// head centroid in metres, origin = physical panel centre, viewer at +Z +float hzAbs = Mathf.Max(Mathf.Abs(hz), 1e-3f); +float targetYaw = FACE_YAW_SIGN * Mathf.Atan2(hx, hzAbs) * Mathf.Rad2Deg; // FACE_YAW_SIGN = -1 +``` + +Typical live values: `hz` ~0.5 m, `hx` ±0.12 m → ±14° of yaw. + +Why not world space: on the **display-centric rig, scale-as-zoom inflates world +units**, so the same head reads as `ipd≈1.2` world units against a physical 0.06 m — +and any reference vector you build from scene objects inherits the scene's layout. +Two world-space attempts failed here: one keyed off `(camera − tiger)`, which +collapses because the tiger's root sits at the camera's x/z; the other picked its +target with `FindAnyObjectByType()` and silently rotated the +**invisible fallback cube** (`TransparentAutoSetup` wires one onto every target +root — the tiger *and* `Cube`). Physical space has neither failure mode. + +Smoothing is time-based exponential (`tau = 0.04 s`), and the heading only chases +while eye tracking is **locked** — warmup positions jitter it. + +Viewer-position sources, all live per-frame: + +| API | Space | Use for | +|-----|-------|---------| +| `DisplayXRNative.displayxr_get_eye_positions` | **physical metres**, panel-centre origin, viewer at +Z | **head-coupled effects — start here** (what the billboard uses) | +| `DisplayXRProvider.TryGetViewerHead` / `TryGetViewerEyes` | Unity **world** (v2.9.1+) | when the effect genuinely needs scene-space coordinates | +| `DisplayXRNative.displayxr_get_stereo_matrices` | per-eye view + proj | cyclopean hit-test, off-axis probes (`KooimaProbe`) | + +Not ported from the reference: the **zone-canvas rebase**, which offsets `hx` by the +window's centre on the panel. Without it the heading references the *panel* centre +rather than the *window* centre — a constant bias when the window is off-centre, not +a tracking failure. + +### On `Camera.GetStereoViewMatrix` (plugin #236) + +[Plugin #236](https://github.com/DisplayXR/displayxr-unity/issues/236) reports it +returning the **same matrix for both eyes** under the provider, which silently +freezes anything built on it. The plugin never writes that cache +(`Camera.SetStereoViewMatrix` went away with the #166 provider migration). But note: +**it does NOT reproduce in a built player here** — `TigerFaceViewer`'s log reads +`eyesEqual=False` on Windows/D3D12, with values matching the provider's exactly. So +the failure is configuration-specific (editor vs player, Unity version), not +universal. Use the physical or provider APIs above and the question doesn't arise. + ## Tiger asset facts - **Rig type: Generic** (not Legacy). Required for `Animator` instead of the deprecated `Animation` component. @@ -90,6 +143,13 @@ After Build And Run for Windows: 6. **W / S** keys → camera (and therefore the tiger relative to the display) push in/out of the display plane. 7. **A / Q / D / E** keys → no effect (locked by `LockToForwardAxis`). 8. Tab → cycles between Main Camera (display rig) and Cam Centric (camera rig); `DragRotateCube` rebinds its listeners automatically. +9. **Head-tracked billboard is ON at startup** (`TigerFaceViewer`). Move your head + left/right in front of the panel: the tiger turns to follow you. The + once-per-second log should read like + `rawHead (-0.115,0.111,0.509)m tracked=True -> targetYaw=12.4 yaw=11.8deg on + cartoon tiger…` — i.e. `hz` ~0.5 m, `hx` ±0.12 m, yaw ±14°, on the **tiger** + (if the target name is `Cube`, it grabbed the invisible fallback fixture). + Press **F** to turn it off and restore the authored rotation. ## Open issues diff --git a/samples/desktop-avatar/Packages/manifest.json b/samples/desktop-avatar/Packages/manifest.json index 2730e95..42327d7 100644 --- a/samples/desktop-avatar/Packages/manifest.json +++ b/samples/desktop-avatar/Packages/manifest.json @@ -1,6 +1,6 @@ { "dependencies": { - "com.displayxr.unity": "https://github.com/DisplayXR/displayxr-unity.git#upm/v2.9.0", + "com.displayxr.unity": "https://github.com/DisplayXR/displayxr-unity.git#upm/v2.9.1", "com.unity.multiplayer.center": "1.0.1", "com.unity.render-pipelines.universal": "17.0.4", "com.unity.ugui": "2.0.0", diff --git a/samples/desktop-avatar/Packages/packages-lock.json b/samples/desktop-avatar/Packages/packages-lock.json index 8373fb5..406c300 100644 --- a/samples/desktop-avatar/Packages/packages-lock.json +++ b/samples/desktop-avatar/Packages/packages-lock.json @@ -1,13 +1,13 @@ { "dependencies": { "com.displayxr.unity": { - "version": "https://github.com/DisplayXR/displayxr-unity.git#upm/v2.9.0", + "version": "https://github.com/DisplayXR/displayxr-unity.git#upm/v2.9.1", "depth": 0, "source": "git", "dependencies": { "com.unity.xr.management": "4.4.0" }, - "hash": "a21c59c2925339f36feb4989fbeae1031f5cc929" + "hash": "416bf4e8018e71afb626d6c8b423cd258b16bd56" }, "com.unity.burst": { "version": "1.8.28", diff --git a/samples/desktop-avatar/README.md b/samples/desktop-avatar/README.md index f8f52d8..5b2a652 100644 --- a/samples/desktop-avatar/README.md +++ b/samples/desktop-avatar/README.md @@ -10,6 +10,21 @@ see-through desktop avatar (cartoon tiger) rendered over the live screen on - **Per-eye foreground clipping** - **`XR_DXR_display_zones`** layout — the avatar is Kooima-projected into a 3D zone, with a **Local2D** speech-bubble zone alongside it +- **Head-tracked billboard** — the tiger turns to face you as you move in front + of the display (yaw-only), driven by the eye tracker + +## Controls + +| Key / input | Effect | +|-------------|--------| +| **F** | Turn the head-tracked billboard **off** (it is on at startup) and restore the tiger's rotation | +| **Left-drag** on the tiger | Rotate it by hand | +| **Right-drag** | Move the application window | +| **Scroll** | Zoom (display rig's virtual display height) | +| **W / S** | Push the tiger in / out of the display plane | +| **Tab** | Cycle rig cameras | +| **Space** | Reset the tiger's orientation | +| **Esc** | Quit | Part of the [displayxr-unity-samples](https://github.com/DisplayXR/displayxr-unity-samples) monorepo — see the [root README](../../README.md) for prerequisites and the