From 180d881e107bec0b6bc7b762f633d555ef7ced37 Mon Sep 17 00:00:00 2001 From: 000x999 <124853841+000x999@users.noreply.github.com> Date: Thu, 20 Aug 2026 05:09:54 -0400 Subject: [PATCH 1/2] Mirror room capture, fixations and remote variables from the WebXR SDK --- README.md | 51 +++++++++++++++++ src/Cognitive3D.ts | 116 ++++++++++++++++++++++++++++++++++++++ src/Cognitive3DContext.ts | 40 ++++++++++++- 3 files changed, 206 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a66321b..495f10b 100644 --- a/README.md +++ b/README.md @@ -13,5 +13,56 @@ In the add-ons and dependencies of Mattercraft, search for `@cognitive3d/three-m * **Quick Setup:** Add the Cognitive3D Manager directly to your scene hierarchy. * **UI Properties Panel:** Easily paste your API keys and Scene data. * **Dynamic Object Tracking:** Select any 3D model in your Mattercraft project and attach the `Cognitive3DDynamicObject` behavior to track positions, rotations, and heatmaps. +* **Room Capture:** Records the participant's real-world room (walls, floors, furniture) as labelled anchors. Toggled with `Enable Room Capture`. +* **Fixations:** Records dispersion-classified fixations from the gaze stream. Toggled with `Enable Fixations`. +* **Remote Variables:** Fetches per-participant variables from the dashboard and exposes them to your own behaviors. * **Scene and Dynamic Object Export:** Press `Shift+E` inside Mattercraft preview to export your environment for the dashboard. Press `Shift+D` to export dynamic objects. * **NOTE** : Ensure the Scene Export toggle is enabled and you save your scene to export data. You can find this setting under the Cognitive3D Behavior component in your scene hierarchy. Disable the toggle after the export is complete. + +## Room Capture, Fixations and Remote Variables + +These three features come from the underlying WebXR SDK and require a version of +`@cognitive3d/analytics` that ships them. Against an older SDK the integration detects +their absence and stays inactive, so nothing breaks. + +### Room Capture + +Room Capture reads the room model the headset already holds, so two conditions apply on +Quest: + +* The session must be **`immersive-ar`**. In `immersive-vr` the runtime reports + `plane-detection` as enabled but never returns any planes. +* The participant must have completed **Space Setup** on the headset. Without an authored + room there is nothing to capture. + +Mattercraft's own XR session does not request the geometry features, so this behavior adds +`plane-detection` and `mesh-detection` to the requested optional features while +`Enable Room Capture` is on. Both are optional, so runtimes without support are unaffected. +Enable debug logging to see which features the runtime granted. + +### Fixations + +Fixations are classified from the gaze stream the Mattercraft adapter already records, so +no extra setup is needed. They are only recorded on headsets that report eye tracking, +which matches the Unity SDK. Set `Allow Fixations Without Eye Tracking` to also classify +fixations from the head-gaze fallback. + +### Remote Variables + +With `Fetch Remote Variables` enabled the SDK fetches the participant's variables when the +session starts. Read them from the shared context: + +```ts +import { Cognitive3DContext } from "@cognitive3d/three-mattercraft"; + +const c3d = this.contextManager.get(Cognitive3DContext); + +c3d.onRemoteVariablesAvailable.addListener(() => { + const difficulty = c3d.getRemoteVariable("difficulty", "normal"); + const showHints = c3d.getRemoteVariable("show_hints", false); +}); +``` + +`getRemoteVariable(name, defaultValue)` returns the default until the fetch resolves, so it +is always safe to call. `listRemoteVariables()` returns everything that resolved, and +`fetchRemoteVariables(identifier)` triggers a fetch manually. diff --git a/src/Cognitive3D.ts b/src/Cognitive3D.ts index 678bf6d..db82c06 100644 --- a/src/Cognitive3D.ts +++ b/src/Cognitive3D.ts @@ -36,6 +36,42 @@ export interface Cognitive3DConstructionProps { * @zdefault false */ enableDebug: boolean; + /** + * @zui + * @zlabel Enable Room Capture + * @zdefault true + */ + enableRoomCapture: boolean; + /** + * @zui + * @zlabel Room Data Limit + * @zdefault 64 + */ + roomDataLimit: number; + /** + * @zui + * @zlabel Enable Fixations + * @zdefault true + */ + enableFixations: boolean; + /** + * @zui + * @zlabel Allow Fixations Without Eye Tracking + * @zdefault false + */ + allowFixationWithoutEyeTracking: boolean; + /** + * @zui + * @zlabel Fixation Data Limit + * @zdefault 256 + */ + fixationDataLimit: number; + /** + * @zui + * @zlabel Fetch Remote Variables + * @zdefault true + */ + autoFetchRemoteVariables: boolean; } @@ -57,6 +93,8 @@ export class Cognitive3D extends Behavior { private sceneContext: ThreeSceneContext; private _xrSession: XRSession | null = null; private _xrSessionEndHandler: (() => void) | null = null; + private _originalRequestSession: ((mode: any, init?: any) => Promise) | null = null; + private _remoteVariablesHandler: (() => void) | null = null; constructor(contextManager: ContextManager, instance: Component, protected constructorProps: Cognitive3DConstructionProps) { super(contextManager, instance); @@ -78,6 +116,12 @@ export class Cognitive3D extends Behavior { APIKey: this.constructorProps.apiKey, LOG: this.constructorProps.enableDebug, gazeTrackingSource: "engine", + enableRoomCapture: this.constructorProps.enableRoomCapture !== false, + roomDataLimit: this.constructorProps.roomDataLimit || 64, + enableFixation: this.constructorProps.enableFixations !== false, + allowFixationWithoutEyeTracking: this.constructorProps.allowFixationWithoutEyeTracking === true, + fixationDataLimit: this.constructorProps.fixationDataLimit || 256, + autoFetchRemoteVariables: this.constructorProps.autoFetchRemoteVariables !== false, allSceneData: [{ sceneId: this.constructorProps.sceneId, sceneName: this.constructorProps.sceneName, @@ -99,6 +143,12 @@ export class Cognitive3D extends Behavior { this.ctx.c3dAdapter = this.c3dAdapter; this.ctx.sceneName = this.constructorProps.sceneName; this.ctx.enableDebug = this.constructorProps.enableDebug; + this.ctx.roomCaptureEnabled = this.constructorProps.enableRoomCapture !== false; + this.ctx.fixationsEnabled = this.constructorProps.enableFixations !== false; + + if (this.ctx.roomCaptureEnabled) { + this.installGeometryFeatures(); + } this.ctx.registerDynamicObject = (b) => this.registerDynamicObject(b); if (this.xrContext) { @@ -124,6 +174,61 @@ export class Cognitive3D extends Behavior { } } + private installGeometryFeatures() { + const xr = (navigator as any).xr; + if (!xr || typeof xr.requestSession !== "function" || this._originalRequestSession) { + return; + } + const original = xr.requestSession.bind(xr); + this._originalRequestSession = original; + xr.requestSession = (mode: any, init?: any) => { + const next = Object.assign({}, init || {}); + const features = Array.isArray(next.optionalFeatures) ? next.optionalFeatures.slice() : []; + ["plane-detection", "mesh-detection"].forEach(feature => { + if (features.indexOf(feature) === -1) features.push(feature); + }); + next.optionalFeatures = features; + return original(mode, next); + }; + } + + private removeGeometryFeatures() { + const xr = (navigator as any).xr; + if (xr && this._originalRequestSession) { + xr.requestSession = this._originalRequestSession; + } + this._originalRequestSession = null; + } + + private setupRemoteVariables() { + const remote = this.c3d && this.c3d.remoteVariables; + if (!remote || typeof remote.onRemoteVariablesAvailable !== "function") { + return; + } + + if (this._remoteVariablesHandler && typeof remote.offRemoteVariablesAvailable === "function") { + remote.offRemoteVariablesAvailable(this._remoteVariablesHandler); + } + + this._remoteVariablesHandler = () => { + this.ctx.remoteVariablesReady = true; + this.ctx.debug(`Cognitive3D: ${this.ctx.listRemoteVariables().length} remote variables available.`); + this.ctx.onRemoteVariablesAvailable.emit(); + }; + + remote.onRemoteVariablesAvailable(this._remoteVariablesHandler); + } + + private reportGeometryFeatures(session: XRSession) { + const granted = (session as any).enabledFeatures ? Array.from((session as any).enabledFeatures) : []; + const hasPlanes = granted.indexOf("plane-detection") !== -1; + const hasMeshes = granted.indexOf("mesh-detection") !== -1; + this.ctx.debug(`Cognitive3D: plane-detection ${hasPlanes ? "granted" : "unavailable"}, mesh-detection ${hasMeshes ? "granted" : "unavailable"}.`); + if (this.ctx.roomCaptureEnabled && !hasPlanes && !hasMeshes) { + console.warn("Cognitive3D: Room Capture is enabled but the runtime granted no geometry features. On Quest this requires an immersive-ar session and a completed Space Setup."); + } + } + public registerDynamicObject(behavior: IDynamicObjectBehavior) { // Add to the internal registry so we can re-initialize on session start this.ctx.trackedBehaviors.add(behavior); @@ -251,6 +356,8 @@ export class Cognitive3D extends Behavior { if (success) { this.ctx.debug("Cognitive3D: Session Started"); + this.reportGeometryFeatures(session); + this.setupRemoteVariables(); const renderer = this.threeContext.renderer as THREE.WebGLRenderer; const scene = this.sceneContext.scene; @@ -463,6 +570,14 @@ export class Cognitive3D extends Behavior { public override dispose() { window.removeEventListener('keydown', this.handleKeyDown); + this.removeGeometryFeatures(); + + const remote = this.c3d && this.c3d.remoteVariables; + if (remote && this._remoteVariablesHandler && typeof remote.offRemoteVariablesAvailable === "function") { + remote.offRemoteVariablesAvailable(this._remoteVariablesHandler); + } + this._remoteVariablesHandler = null; + if (this._xrSession && this._xrSessionEndHandler) { this._xrSession.removeEventListener("end", this._xrSessionEndHandler); } @@ -478,6 +593,7 @@ export class Cognitive3D extends Behavior { this.ctx.registerDynamicObject = null; this.ctx.trackedBehaviors.clear(); this.ctx.registeredWithSDK.clear(); + this.ctx.remoteVariablesReady = false; return super.dispose(); } diff --git a/src/Cognitive3DContext.ts b/src/Cognitive3DContext.ts index dec8ee0..3602a1a 100644 --- a/src/Cognitive3DContext.ts +++ b/src/Cognitive3DContext.ts @@ -1,4 +1,4 @@ -import { Context, ContextManager } from "@zcomponent/core"; +import { Context, ContextManager, Event } from "@zcomponent/core"; import * as THREE from "three"; export interface IDynamicObjectBehavior { @@ -18,6 +18,10 @@ export class Cognitive3DContext extends Context { public pendingRegistrations: IDynamicObjectBehavior[] = []; public sceneName: string = ""; public enableDebug: boolean = false; + public roomCaptureEnabled: boolean = false; + public fixationsEnabled: boolean = false; + public remoteVariablesReady: boolean = false; + public onRemoteVariablesAvailable: Event<[]> = new Event(); /** Set by the Cognitive3D behavior so DynamicObjects can trigger full SDK registration. */ public registerDynamicObject: ((behavior: IDynamicObjectBehavior) => void) | null = null; @@ -46,12 +50,46 @@ export class Cognitive3DContext extends Context { this.c3d.customEvent.send(category, position, properties); } + public getRemoteVariable(name: string, defaultValue: T): T { + const remote = this.c3d && this.c3d.remoteVariables; + if (!remote || typeof remote.getValue !== "function") return defaultValue; + try { + return remote.getValue(name, defaultValue) as T; + } catch (e) { + return defaultValue; + } + } + + public listRemoteVariables(): any[] { + const remote = this.c3d && this.c3d.remoteVariables; + if (!remote || typeof remote.listAllVariables !== "function") return []; + try { + return remote.listAllVariables(); + } catch (e) { + return []; + } + } + + public fetchRemoteVariables(identifier?: string): Promise { + const remote = this.c3d && this.c3d.remoteVariables; + if (!remote || typeof remote.fetchVariables !== "function") return Promise.resolve(false); + try { + return Promise.resolve(remote.fetchVariables(identifier)); + } catch (e) { + return Promise.resolve(false); + } + } + dispose() { this.c3d = null; this.c3dAdapter = null; this.trackedBehaviors.clear(); this.registeredWithSDK.clear(); this.pendingRegistrations = []; + this.roomCaptureEnabled = false; + this.fixationsEnabled = false; + this.remoteVariablesReady = false; + this.onRemoteVariablesAvailable.clearListeners(); return super.dispose(); } } From 537211f868cdc8a42a6bc0f65d6b928119f5a2a4 Mon Sep 17 00:00:00 2001 From: 000x999 <124853841+000x999@users.noreply.github.com> Date: Thu, 20 Aug 2026 05:12:59 -0400 Subject: [PATCH 2/2] Remove room capture, fixations and remote variables sections from README --- README.md | 51 --------------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/README.md b/README.md index 495f10b..a66321b 100644 --- a/README.md +++ b/README.md @@ -13,56 +13,5 @@ In the add-ons and dependencies of Mattercraft, search for `@cognitive3d/three-m * **Quick Setup:** Add the Cognitive3D Manager directly to your scene hierarchy. * **UI Properties Panel:** Easily paste your API keys and Scene data. * **Dynamic Object Tracking:** Select any 3D model in your Mattercraft project and attach the `Cognitive3DDynamicObject` behavior to track positions, rotations, and heatmaps. -* **Room Capture:** Records the participant's real-world room (walls, floors, furniture) as labelled anchors. Toggled with `Enable Room Capture`. -* **Fixations:** Records dispersion-classified fixations from the gaze stream. Toggled with `Enable Fixations`. -* **Remote Variables:** Fetches per-participant variables from the dashboard and exposes them to your own behaviors. * **Scene and Dynamic Object Export:** Press `Shift+E` inside Mattercraft preview to export your environment for the dashboard. Press `Shift+D` to export dynamic objects. * **NOTE** : Ensure the Scene Export toggle is enabled and you save your scene to export data. You can find this setting under the Cognitive3D Behavior component in your scene hierarchy. Disable the toggle after the export is complete. - -## Room Capture, Fixations and Remote Variables - -These three features come from the underlying WebXR SDK and require a version of -`@cognitive3d/analytics` that ships them. Against an older SDK the integration detects -their absence and stays inactive, so nothing breaks. - -### Room Capture - -Room Capture reads the room model the headset already holds, so two conditions apply on -Quest: - -* The session must be **`immersive-ar`**. In `immersive-vr` the runtime reports - `plane-detection` as enabled but never returns any planes. -* The participant must have completed **Space Setup** on the headset. Without an authored - room there is nothing to capture. - -Mattercraft's own XR session does not request the geometry features, so this behavior adds -`plane-detection` and `mesh-detection` to the requested optional features while -`Enable Room Capture` is on. Both are optional, so runtimes without support are unaffected. -Enable debug logging to see which features the runtime granted. - -### Fixations - -Fixations are classified from the gaze stream the Mattercraft adapter already records, so -no extra setup is needed. They are only recorded on headsets that report eye tracking, -which matches the Unity SDK. Set `Allow Fixations Without Eye Tracking` to also classify -fixations from the head-gaze fallback. - -### Remote Variables - -With `Fetch Remote Variables` enabled the SDK fetches the participant's variables when the -session starts. Read them from the shared context: - -```ts -import { Cognitive3DContext } from "@cognitive3d/three-mattercraft"; - -const c3d = this.contextManager.get(Cognitive3DContext); - -c3d.onRemoteVariablesAvailable.addListener(() => { - const difficulty = c3d.getRemoteVariable("difficulty", "normal"); - const showHints = c3d.getRemoteVariable("show_hints", false); -}); -``` - -`getRemoteVariable(name, defaultValue)` returns the default until the fetch resolves, so it -is always safe to call. `listRemoteVariables()` returns everything that resolved, and -`fetchRemoteVariables(identifier)` triggers a fetch manually.