From 74d19f7f30aa57bc65415ac597c62a7f41daae49 Mon Sep 17 00:00:00 2001 From: amariichi <68761912+amariichi@users.noreply.github.com> Date: Tue, 24 Mar 2026 12:10:38 +0900 Subject: [PATCH 01/11] Add phase-1 pinhole reprojection experiment --- backend/routers/stream.py | 16 +++++++++++++--- backend/video/io.py | 24 +++++++++++++++++++++++- webapp/src/app.ts | 17 ++++++++++++++++- webapp/src/network/depthClient.ts | 10 ++++++++++ webapp/src/render/projection.ts | 9 +++++++++ webapp/src/render/scene.ts | 7 +++++++ webapp/src/render/shaders.ts | 13 +++++++++++-- webapp/src/state/playerStore.ts | 1 + webapp/src/types.ts | 3 +++ webapp/src/ui/controls.ts | 6 ++++++ webapp/src/xrTest.ts | 20 +++++++++++++++++--- 11 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 webapp/src/render/projection.ts diff --git a/backend/routers/stream.py b/backend/routers/stream.py index 77d9510..e85be97 100644 --- a/backend/routers/stream.py +++ b/backend/routers/stream.py @@ -13,6 +13,7 @@ from backend.models.depth_model import get_depth_model from backend.utils.packets import pack_depth_payload from backend.utils.depth_ops import downsample_depth +from backend.video.io import EndOfStreamError from backend.video.session import DepthFrame, get_session_manager, SessionManager from backend.config import get_settings from backend.utils.queues import DroppingQueue @@ -50,6 +51,14 @@ async def depth_stream( stats = StatisticsCollector() + async def cancel_pending(*tasks: asyncio.Task) -> None: + pending = [task for task in tasks if task is not None and not task.done()] + for task in pending: + task.cancel() + if pending: + with suppress(Exception): + await asyncio.gather(*pending, return_exceptions=True) + async def receiver() -> None: try: while True: @@ -122,7 +131,7 @@ async def process_request_pipeline(request: dict, session) -> tuple[bytes | None timings["decode_s"] = decode_s # print(f"[Backend] Processing: {time_ms}ms. QueueWait: {timings['queue_wait_s']:.3f}s. Decode: {timings['decode_s']:.3f}s") - except StopIteration: + except EndOfStreamError: # EOF return None, timings except Exception as e: @@ -227,9 +236,9 @@ async def processor() -> None: get_task = asyncio.create_task(request_queue.get()) stop_task = asyncio.create_task(stop.wait()) done, _ = await asyncio.wait([get_task, stop_task], return_when=asyncio.FIRST_COMPLETED) + await cancel_pending(get_task, stop_task) if stop_task in done: - get_task.cancel() break request = get_task.result() @@ -275,9 +284,9 @@ async def sender() -> None: get_task = asyncio.create_task(send_queue.get()) stop_task = asyncio.create_task(stop.wait()) done, _ = await asyncio.wait([get_task, stop_task], return_when=asyncio.FIRST_COMPLETED) + await cancel_pending(get_task, stop_task) if stop_task in done: - get_task.cancel() break task = get_task.result() @@ -311,6 +320,7 @@ async def sender() -> None: int(timings.get("inflight_used", 0)), ) except Exception: + stop.set() break # Socket closed except asyncio.CancelledError: break diff --git a/backend/video/io.py b/backend/video/io.py index c54530a..28df234 100644 --- a/backend/video/io.py +++ b/backend/video/io.py @@ -14,6 +14,23 @@ from backend.utils.frame_info import FrameInfo, frame_info_from_av +class EndOfStreamError(Exception): + """Raised when the decoder reaches EOF for a requested timestamp.""" + + +def _is_av_eof(exc: Exception) -> bool: + av_error_mod = getattr(av, "error", None) + eof_types = tuple( + cls + for cls in ( + getattr(av, "EOFError", None), + getattr(av_error_mod, "EOFError", None), + ) + if isinstance(cls, type) + ) + return isinstance(exc, eof_types) if eof_types else False + + @dataclass(frozen=True) class VideoMetadata: width: int @@ -115,7 +132,12 @@ def _advance_to(self, time_ms: float) -> tuple[np.ndarray, FrameInfo]: frame = next(self._frame_iter) except StopIteration as exc: # pragma: no cover - EOF self._frame_iter = None - raise StopIteration from exc + raise EndOfStreamError from exc + except Exception as exc: # pragma: no cover - decoder EOF varies by PyAV build + if _is_av_eof(exc): + self._frame_iter = None + raise EndOfStreamError from exc + raise info = frame_info_from_av(frame) frames_examined += 1 actual_time = info.time_ms if info.time_ms >= 0 else time_ms diff --git a/webapp/src/app.ts b/webapp/src/app.ts index 4c51ac4..7b4ef26 100644 --- a/webapp/src/app.ts +++ b/webapp/src/app.ts @@ -20,6 +20,7 @@ export class VideoDepthApp { private depthBuffer = new DepthBuffer(); private lastAspect = 0; private rawXR: RawXRTest | null = null; + private suppressSeekRefresh = false; constructor(root: HTMLElement) { // ... (existing constructor code) @@ -31,6 +32,8 @@ export class VideoDepthApp { video.preload = 'metadata'; root.querySelector('#video-container')?.appendChild(video); this.videoEl = video; + this.videoEl.addEventListener('ended', () => this.handleVideoEnded()); + this.videoEl.addEventListener('seeked', () => this.handleVideoSeeked()); usePlayerStore.getState().setVideo(video); const canvasContainer = root.querySelector('#canvas-container') as HTMLElement; @@ -188,7 +191,6 @@ export class VideoDepthApp { this.startRenderLoop(); // Restart render loop }; this.videoEl.addEventListener('loadeddata', onLoaded); - this.videoEl.addEventListener('ended', () => this.handleVideoEnded()); this.videoEl.play(); } @@ -396,6 +398,7 @@ export class VideoDepthApp { private handleVideoEnded(): void { // Reset playback position and depth buffers so replay works this.depthBuffer.clear(); + this.suppressSeekRefresh = true; this.videoEl.currentTime = 0; // Refresh depth stream connection to drop pending/inflight safely if (this.depthClient) { @@ -405,6 +408,18 @@ export class VideoDepthApp { // Let the user press play again; depth polling loop will pick up from t=0 } + private handleVideoSeeked(): void { + if (this.suppressSeekRefresh) { + this.suppressSeekRefresh = false; + return; + } + if (!this.depthClient || !this.currentSession) return; + // Drop stale future requests and frames after explicit timeline jumps. + this.depthBuffer.clear(); + this.depthClient.close(); + this.depthClient.connect(); + } + private healthIntervalId: number | null = null; private startHealthReporting(): void { diff --git a/webapp/src/network/depthClient.ts b/webapp/src/network/depthClient.ts index 6ccf986..7a65302 100644 --- a/webapp/src/network/depthClient.ts +++ b/webapp/src/network/depthClient.ts @@ -107,8 +107,18 @@ export class DepthClient { this.socket.close(); this.socket = null; } + this.resetStreamState(); + } + + resetStreamState(): void { this.pending = []; this.inflight = 0; + this.droppedFrames = 0; + this.lastTimestampMs = -1; + this.requestTimes.clear(); + this.rtt = 0; + this.lastArrival = 0; + this.arrivalDeltas = []; } requestDepth(timeMs: number): void { diff --git a/webapp/src/render/projection.ts b/webapp/src/render/projection.ts new file mode 100644 index 0000000..b339c7a --- /dev/null +++ b/webapp/src/render/projection.ts @@ -0,0 +1,9 @@ +import type { ViewerControls } from '../types'; + +export function getProjectionMix(controls: ViewerControls): number { + return controls.projectionMode === 'pinhole' ? 1.0 : 0.0; +} + +export function getTanHalfFovY(controls: ViewerControls): number { + return Math.tan((controls.fovY * Math.PI) / 360); +} diff --git a/webapp/src/render/scene.ts b/webapp/src/render/scene.ts index 57c9a05..c032cf6 100644 --- a/webapp/src/render/scene.ts +++ b/webapp/src/render/scene.ts @@ -3,6 +3,7 @@ import * as THREE from 'three'; // VRButton unused (RawXRに統一) import type { DepthFrame, ViewerControls } from '../types'; import { createGridGeometry } from './mesh'; +import { getProjectionMix, getTanHalfFovY } from './projection'; import { fragmentShader, vertexShader } from './shaders'; import { perfStats } from '../utils/perfStats'; @@ -229,6 +230,8 @@ export class RenderScene { uniforms.zGamma.value = this.currentControls.zGamma; uniforms.zMaxClip.value = this.currentControls.zMaxClip; uniforms.planeScale.value = this.currentControls.planeScale; + uniforms.projectionMix.value = getProjectionMix(this.currentControls); + uniforms.tanHalfFovY.value = getTanHalfFovY(this.currentControls); this.mesh.position.y = this.currentControls.yOffset; } } @@ -268,6 +271,8 @@ export class RenderScene { uniforms.zGamma.value = controls.zGamma; uniforms.zMaxClip.value = controls.zMaxClip; uniforms.planeScale.value = controls.planeScale; + uniforms.projectionMix.value = getProjectionMix(controls); + uniforms.tanHalfFovY.value = getTanHalfFovY(controls); const yOffset = this.renderer.xr.isPresenting ? controls.yOffset + this.vrYOffset : controls.yOffset; this.mesh.position.y = yOffset; perfStats.add('scene.updateDepth', performance.now() - start); @@ -296,6 +301,8 @@ export class RenderScene { zGamma: { value: controls.zGamma }, zMaxClip: { value: controls.zMaxClip }, planeScale: { value: controls.planeScale }, + projectionMix: { value: getProjectionMix(controls) }, + tanHalfFovY: { value: getTanHalfFovY(controls) }, }, vertexShader, fragmentShader, diff --git a/webapp/src/render/shaders.ts b/webapp/src/render/shaders.ts index 59c20e4..3ce8d3e 100644 --- a/webapp/src/render/shaders.ts +++ b/webapp/src/render/shaders.ts @@ -7,6 +7,8 @@ export const vertexShader = /* glsl */ ` uniform float zGamma; uniform float zMaxClip; uniform float planeScale; + uniform float projectionMix; + uniform float tanHalfFovY; varying vec2 vUv; varying vec2 vSampleUv; varying vec3 vNormal; @@ -26,8 +28,15 @@ export const vertexShader = /* glsl */ ` // non-zero bias easily saturates to a constant Z and the mesh appears flat. float zDepth = clamp(depth * zScale, 0.0, zMaxClip); float z = zDepth + zBias; - float x = (0.5 - vUv.x) * aspect * planeScale; - float y = (0.5 - vUv.y) * planeScale; + float reliefX = (0.5 - vUv.x) * aspect * planeScale; + float reliefY = (0.5 - vUv.y) * planeScale; + // For pinhole mode, normalize the existing planeScale so the default value + // stays close to a unit display scale. + float pinholeSpread = (2.0 * tanHalfFovY) * (0.5 * planeScale); + float pinholeX = (0.5 - vUv.x) * aspect * pinholeSpread * z; + float pinholeY = (0.5 - vUv.y) * pinholeSpread * z; + float x = mix(reliefX, pinholeX, projectionMix); + float y = mix(reliefY, pinholeY, projectionMix); vec4 displaced = vec4(x, y, -z, 1.0); gl_Position = projectionMatrix * modelViewMatrix * displaced; } diff --git a/webapp/src/state/playerStore.ts b/webapp/src/state/playerStore.ts index e4856e8..4972e2a 100644 --- a/webapp/src/state/playerStore.ts +++ b/webapp/src/state/playerStore.ts @@ -18,6 +18,7 @@ export interface PlayerState { } const defaultControls: ViewerControls = { + projectionMode: 'relief', targetTriangles: 200_000, fovY: 50, zScale: 1.0, diff --git a/webapp/src/types.ts b/webapp/src/types.ts index dbc5483..6237316 100644 --- a/webapp/src/types.ts +++ b/webapp/src/types.ts @@ -42,7 +42,10 @@ export interface PerfSettings { autoLead: boolean; } +export type ProjectionMode = 'relief' | 'pinhole'; + export interface ViewerControls { + projectionMode: ProjectionMode; targetTriangles: number; fovY: number; zScale: number; diff --git a/webapp/src/ui/controls.ts b/webapp/src/ui/controls.ts index d18c9ea..98912f7 100644 --- a/webapp/src/ui/controls.ts +++ b/webapp/src/ui/controls.ts @@ -41,6 +41,12 @@ export class ControlPanel { private mountSliders(): void { const store = usePlayerStore.getState(); const folder = this.depthGui; // no extra nesting + folder + .add(store.viewerControls, 'projectionMode', { Relief: 'relief', Pinhole: 'pinhole' }) + .name('Projection') + .onChange((value: 'relief' | 'pinhole') => { + usePlayerStore.getState().updateControls({ projectionMode: value }); + }); folder.add(store.viewerControls, 'targetTriangles', 50_000, 300_000, 10_000).name('Target tris').onChange((value: number) => { usePlayerStore.getState().updateControls({ targetTriangles: value }); }); diff --git a/webapp/src/xrTest.ts b/webapp/src/xrTest.ts index 54f706e..a20a7ed 100644 --- a/webapp/src/xrTest.ts +++ b/webapp/src/xrTest.ts @@ -3,6 +3,7 @@ import * as THREE from 'three'; import { drawThreeStereo } from './xrThreeBridge'; import type { DepthFrame, ViewerControls } from './types'; +import { getProjectionMix, getTanHalfFovY } from './render/projection'; import { DepthBuffer } from './utils/depthBuffer'; type RawXRMode = 'quad' | 'mesh' | 'three'; @@ -25,6 +26,8 @@ interface MeshState { zGamma: WebGLUniformLocation | null; zMaxClip: WebGLUniformLocation | null; planeScale: WebGLUniformLocation | null; + projectionMix: WebGLUniformLocation | null; + tanHalfFovY: WebGLUniformLocation | null; }; } @@ -568,6 +571,8 @@ export class RawXRTest { gl.uniform1f(s.mesh.uniforms.zGamma, controls.zGamma); gl.uniform1f(s.mesh.uniforms.zMaxClip, controls.zMaxClip); gl.uniform1f(s.mesh.uniforms.planeScale, controls.planeScale); + gl.uniform1f(s.mesh.uniforms.projectionMix, getProjectionMix(controls)); + gl.uniform1f(s.mesh.uniforms.tanHalfFovY, getTanHalfFovY(controls)); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, s.mesh.depthTex); @@ -667,6 +672,7 @@ export class RawXRTest { private getControls(): ViewerControls { const fallback: ViewerControls = { + projectionMode: 'relief', targetTriangles: 8000, fovY: 60, zScale: 1.0, @@ -703,6 +709,8 @@ export class RawXRTest { uniform float uZGamma;\n uniform float uZMaxClip;\n uniform float uPlaneScale;\n + uniform float uProjectionMix;\n + uniform float uTanHalfFovY;\n out vec2 vUv;\n void main(){\n // Keep video座標は左→右、上→下。サンプルは上下反転のみ。 @@ -715,9 +723,13 @@ export class RawXRTest { // the mesh looks flat. float zDepth = clamp(depth * uZScale, 0.0, uZMaxClip); float z = zDepth + uZBias; - // 右手系: +X が右。元は符号が逆で背面を向いていた。\n - float x = (aUv.x - 0.5) * uAspect * uPlaneScale;\n - float y = (0.5 - aUv.y) * uPlaneScale;\n + float reliefX = (aUv.x - 0.5) * uAspect * uPlaneScale;\n + float reliefY = (0.5 - aUv.y) * uPlaneScale;\n + float pinholeSpread = (2.0 * uTanHalfFovY) * (0.5 * uPlaneScale);\n + float pinholeX = (aUv.x - 0.5) * uAspect * pinholeSpread * z;\n + float pinholeY = (0.5 - aUv.y) * pinholeSpread * z;\n + float x = mix(reliefX, pinholeX, uProjectionMix);\n + float y = mix(reliefY, pinholeY, uProjectionMix);\n vec4 local = vec4(x, y, -z, 1.0);\n gl_Position = uViewProj * uModel * local;\n }`; @@ -739,6 +751,8 @@ export class RawXRTest { zGamma: gl.getUniformLocation(prog, 'uZGamma'), zMaxClip: gl.getUniformLocation(prog, 'uZMaxClip'), planeScale: gl.getUniformLocation(prog, 'uPlaneScale'), + projectionMix: gl.getUniformLocation(prog, 'uProjectionMix'), + tanHalfFovY: gl.getUniformLocation(prog, 'uTanHalfFovY'), }; gl.useProgram(prog); const depthLoc = gl.getUniformLocation(prog, 'uDepthTex'); From 45e21cbd937307b2631a7ee53172a37c7242812e Mon Sep 17 00:00:00 2001 From: amariichi <68761912+amariichi@users.noreply.github.com> Date: Tue, 24 Mar 2026 19:27:58 +0900 Subject: [PATCH 02/11] Add phase-2 source FOV and edge cutting --- webapp/src/render/projection.ts | 8 ++++-- webapp/src/render/scene.ts | 12 ++++++--- webapp/src/render/shaders.ts | 39 +++++++++++++++++++-------- webapp/src/state/playerStore.ts | 2 ++ webapp/src/types.ts | 2 ++ webapp/src/ui/controls.ts | 8 +++++- webapp/src/xrTest.ts | 48 ++++++++++++++++++++++++++------- 7 files changed, 91 insertions(+), 28 deletions(-) diff --git a/webapp/src/render/projection.ts b/webapp/src/render/projection.ts index b339c7a..7e11e66 100644 --- a/webapp/src/render/projection.ts +++ b/webapp/src/render/projection.ts @@ -4,6 +4,10 @@ export function getProjectionMix(controls: ViewerControls): number { return controls.projectionMode === 'pinhole' ? 1.0 : 0.0; } -export function getTanHalfFovY(controls: ViewerControls): number { - return Math.tan((controls.fovY * Math.PI) / 360); +export function getTanHalfSourceFovY(controls: ViewerControls): number { + return Math.tan((controls.sourceFovY * Math.PI) / 360); +} + +export function getEffectiveEdgeDiscardThreshold(controls: ViewerControls): number { + return controls.projectionMode === 'pinhole' ? controls.edgeDiscardThreshold : 0.0; } diff --git a/webapp/src/render/scene.ts b/webapp/src/render/scene.ts index c032cf6..2dd8ee1 100644 --- a/webapp/src/render/scene.ts +++ b/webapp/src/render/scene.ts @@ -3,7 +3,7 @@ import * as THREE from 'three'; // VRButton unused (RawXRに統一) import type { DepthFrame, ViewerControls } from '../types'; import { createGridGeometry } from './mesh'; -import { getProjectionMix, getTanHalfFovY } from './projection'; +import { getEffectiveEdgeDiscardThreshold, getProjectionMix, getTanHalfSourceFovY } from './projection'; import { fragmentShader, vertexShader } from './shaders'; import { perfStats } from '../utils/perfStats'; @@ -231,7 +231,8 @@ export class RenderScene { uniforms.zMaxClip.value = this.currentControls.zMaxClip; uniforms.planeScale.value = this.currentControls.planeScale; uniforms.projectionMix.value = getProjectionMix(this.currentControls); - uniforms.tanHalfFovY.value = getTanHalfFovY(this.currentControls); + uniforms.tanHalfSourceFovY.value = getTanHalfSourceFovY(this.currentControls); + uniforms.edgeDiscardThreshold.value = getEffectiveEdgeDiscardThreshold(this.currentControls); this.mesh.position.y = this.currentControls.yOffset; } } @@ -265,6 +266,7 @@ export class RenderScene { this.mesh.material.uniforms.depthTexture.value = this.depthTexture; } const uniforms = this.mesh.material.uniforms; + uniforms.depthSize.value.set(frame.width, frame.height); uniforms.aspect.value = frame.width / frame.height; uniforms.zScale.value = controls.zScale; uniforms.zBias.value = controls.zBias; @@ -272,7 +274,8 @@ export class RenderScene { uniforms.zMaxClip.value = controls.zMaxClip; uniforms.planeScale.value = controls.planeScale; uniforms.projectionMix.value = getProjectionMix(controls); - uniforms.tanHalfFovY.value = getTanHalfFovY(controls); + uniforms.tanHalfSourceFovY.value = getTanHalfSourceFovY(controls); + uniforms.edgeDiscardThreshold.value = getEffectiveEdgeDiscardThreshold(controls); const yOffset = this.renderer.xr.isPresenting ? controls.yOffset + this.vrYOffset : controls.yOffset; this.mesh.position.y = yOffset; perfStats.add('scene.updateDepth', performance.now() - start); @@ -302,7 +305,8 @@ export class RenderScene { zMaxClip: { value: controls.zMaxClip }, planeScale: { value: controls.planeScale }, projectionMix: { value: getProjectionMix(controls) }, - tanHalfFovY: { value: getTanHalfFovY(controls) }, + tanHalfSourceFovY: { value: getTanHalfSourceFovY(controls) }, + edgeDiscardThreshold: { value: getEffectiveEdgeDiscardThreshold(controls) }, }, vertexShader, fragmentShader, diff --git a/webapp/src/render/shaders.ts b/webapp/src/render/shaders.ts index 3ce8d3e..14d11d2 100644 --- a/webapp/src/render/shaders.ts +++ b/webapp/src/render/shaders.ts @@ -8,31 +8,43 @@ export const vertexShader = /* glsl */ ` uniform float zMaxClip; uniform float planeScale; uniform float projectionMix; - uniform float tanHalfFovY; + uniform float tanHalfSourceFovY; varying vec2 vUv; varying vec2 vSampleUv; - varying vec3 vNormal; + varying float vEdgeMetric; float readDepth(vec2 uv) { - vec2 texel = uv; - return texture(depthTexture, texel).r; + return texture(depthTexture, uv).r; + } + + float shapeDepth(float rawDepth) { + float depth = pow(max(rawDepth, 0.0), zGamma); + return clamp(depth * zScale, 0.0, zMaxClip); + } + + float relativeDiff(float a, float b) { + return abs(a - b) / max(max(a, b), 1e-3); } void main() { vUv = uv; vSampleUv = vec2(1.0 - uv.x, 1.0 - uv.y); - float depth = readDepth(vSampleUv); - depth = pow(max(depth, 0.0), zGamma); - // Apply clipping to the depth-derived displacement only. - // Z Bias is a global offset and should not participate in clipping, otherwise - // non-zero bias easily saturates to a constant Z and the mesh appears flat. - float zDepth = clamp(depth * zScale, 0.0, zMaxClip); + vec2 texel = vec2(1.0) / max(depthSize, vec2(1.0)); + float zDepth = shapeDepth(readDepth(vSampleUv)); float z = zDepth + zBias; + float leftDepth = shapeDepth(readDepth(vSampleUv + vec2(-texel.x, 0.0))); + float rightDepth = shapeDepth(readDepth(vSampleUv + vec2(texel.x, 0.0))); + float upDepth = shapeDepth(readDepth(vSampleUv + vec2(0.0, texel.y))); + float downDepth = shapeDepth(readDepth(vSampleUv + vec2(0.0, -texel.y))); + vEdgeMetric = max( + max(relativeDiff(zDepth, leftDepth), relativeDiff(zDepth, rightDepth)), + max(relativeDiff(zDepth, upDepth), relativeDiff(zDepth, downDepth)) + ); float reliefX = (0.5 - vUv.x) * aspect * planeScale; float reliefY = (0.5 - vUv.y) * planeScale; // For pinhole mode, normalize the existing planeScale so the default value // stays close to a unit display scale. - float pinholeSpread = (2.0 * tanHalfFovY) * (0.5 * planeScale); + float pinholeSpread = (2.0 * tanHalfSourceFovY) * (0.5 * planeScale); float pinholeX = (0.5 - vUv.x) * aspect * pinholeSpread * z; float pinholeY = (0.5 - vUv.y) * pinholeSpread * z; float x = mix(reliefX, pinholeX, projectionMix); @@ -44,10 +56,15 @@ export const vertexShader = /* glsl */ ` export const fragmentShader = /* glsl */ ` uniform sampler2D videoTexture; + uniform float edgeDiscardThreshold; varying vec2 vUv; varying vec2 vSampleUv; + varying float vEdgeMetric; void main() { + if (edgeDiscardThreshold > 0.0 && vEdgeMetric > edgeDiscardThreshold) { + discard; + } vec4 color = texture(videoTexture, vSampleUv); gl_FragColor = color; } diff --git a/webapp/src/state/playerStore.ts b/webapp/src/state/playerStore.ts index 4972e2a..407c58c 100644 --- a/webapp/src/state/playerStore.ts +++ b/webapp/src/state/playerStore.ts @@ -21,10 +21,12 @@ const defaultControls: ViewerControls = { projectionMode: 'relief', targetTriangles: 200_000, fovY: 50, + sourceFovY: 50, zScale: 1.0, zBias: 0.0, zGamma: 1.0, zMaxClip: 50, + edgeDiscardThreshold: 0.2, planeScale: 2.0, yOffset: 1.2, }; diff --git a/webapp/src/types.ts b/webapp/src/types.ts index 6237316..6063b50 100644 --- a/webapp/src/types.ts +++ b/webapp/src/types.ts @@ -48,10 +48,12 @@ export interface ViewerControls { projectionMode: ProjectionMode; targetTriangles: number; fovY: number; + sourceFovY: number; zScale: number; zBias: number; zGamma: number; zMaxClip: number; + edgeDiscardThreshold: number; planeScale: number; yOffset: number; } diff --git a/webapp/src/ui/controls.ts b/webapp/src/ui/controls.ts index 98912f7..0b32b88 100644 --- a/webapp/src/ui/controls.ts +++ b/webapp/src/ui/controls.ts @@ -50,9 +50,12 @@ export class ControlPanel { folder.add(store.viewerControls, 'targetTriangles', 50_000, 300_000, 10_000).name('Target tris').onChange((value: number) => { usePlayerStore.getState().updateControls({ targetTriangles: value }); }); - folder.add(store.viewerControls, 'fovY', 30, 90, 1).name('FOV Y').onChange((value: number) => { + folder.add(store.viewerControls, 'fovY', 30, 90, 1).name('View FOV Y').onChange((value: number) => { usePlayerStore.getState().updateControls({ fovY: value }); }); + folder.add(store.viewerControls, 'sourceFovY', 30, 100, 1).name('Source FOV Y').onChange((value: number) => { + usePlayerStore.getState().updateControls({ sourceFovY: value }); + }); folder.add(store.viewerControls, 'zScale', 0.5, 5.0, 0.05).name('Z Scale').onChange((value: number) => { usePlayerStore.getState().updateControls({ zScale: value }); }); @@ -65,6 +68,9 @@ export class ControlPanel { folder.add(store.viewerControls, 'zMaxClip', 5, 100, 1).name('Z Max Clip').onChange((value: number) => { usePlayerStore.getState().updateControls({ zMaxClip: value }); }); + folder.add(store.viewerControls, 'edgeDiscardThreshold', 0.0, 0.6, 0.01).name('Edge Cut').onChange((value: number) => { + usePlayerStore.getState().updateControls({ edgeDiscardThreshold: value }); + }); folder.add(store.viewerControls, 'planeScale', 0.5, 3.5, 0.1).name('Plane Scale').onChange((value: number) => { usePlayerStore.getState().updateControls({ planeScale: value }); }); diff --git a/webapp/src/xrTest.ts b/webapp/src/xrTest.ts index a20a7ed..2f73721 100644 --- a/webapp/src/xrTest.ts +++ b/webapp/src/xrTest.ts @@ -3,7 +3,7 @@ import * as THREE from 'three'; import { drawThreeStereo } from './xrThreeBridge'; import type { DepthFrame, ViewerControls } from './types'; -import { getProjectionMix, getTanHalfFovY } from './render/projection'; +import { getEffectiveEdgeDiscardThreshold, getProjectionMix, getTanHalfSourceFovY } from './render/projection'; import { DepthBuffer } from './utils/depthBuffer'; type RawXRMode = 'quad' | 'mesh' | 'three'; @@ -20,6 +20,7 @@ interface MeshState { uniforms: { viewProj: WebGLUniformLocation | null; model: WebGLUniformLocation | null; + depthSize: WebGLUniformLocation | null; aspect: WebGLUniformLocation | null; zScale: WebGLUniformLocation | null; zBias: WebGLUniformLocation | null; @@ -27,7 +28,8 @@ interface MeshState { zMaxClip: WebGLUniformLocation | null; planeScale: WebGLUniformLocation | null; projectionMix: WebGLUniformLocation | null; - tanHalfFovY: WebGLUniformLocation | null; + tanHalfSourceFovY: WebGLUniformLocation | null; + edgeDiscardThreshold: WebGLUniformLocation | null; }; } @@ -565,6 +567,7 @@ export class RawXRTest { gl.useProgram(s.mesh.prog); gl.uniformMatrix4fv(s.mesh.uniforms.viewProj, false, viewProj.elements as unknown as Float32List); gl.uniformMatrix4fv(s.mesh.uniforms.model, false, model.elements as unknown as Float32List); + gl.uniform2f(s.mesh.uniforms.depthSize, depth.width, depth.height); gl.uniform1f(s.mesh.uniforms.aspect, depth.width / depth.height); gl.uniform1f(s.mesh.uniforms.zScale, controls.zScale); gl.uniform1f(s.mesh.uniforms.zBias, controls.zBias); @@ -572,7 +575,8 @@ export class RawXRTest { gl.uniform1f(s.mesh.uniforms.zMaxClip, controls.zMaxClip); gl.uniform1f(s.mesh.uniforms.planeScale, controls.planeScale); gl.uniform1f(s.mesh.uniforms.projectionMix, getProjectionMix(controls)); - gl.uniform1f(s.mesh.uniforms.tanHalfFovY, getTanHalfFovY(controls)); + gl.uniform1f(s.mesh.uniforms.tanHalfSourceFovY, getTanHalfSourceFovY(controls)); + gl.uniform1f(s.mesh.uniforms.edgeDiscardThreshold, getEffectiveEdgeDiscardThreshold(controls)); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, s.mesh.depthTex); @@ -675,10 +679,12 @@ export class RawXRTest { projectionMode: 'relief', targetTriangles: 8000, fovY: 60, + sourceFovY: 60, zScale: 1.0, zBias: 0.0, zGamma: 1.0, zMaxClip: 5.0, + edgeDiscardThreshold: 0.2, planeScale: 1.0, yOffset: 0.0, }; @@ -703,6 +709,7 @@ export class RawXRTest { uniform mat4 uViewProj;\n uniform mat4 uModel;\n uniform sampler2D uDepthTex;\n + uniform vec2 uDepthSize;\n uniform float uAspect;\n uniform float uZScale;\n uniform float uZBias;\n @@ -710,22 +717,36 @@ export class RawXRTest { uniform float uZMaxClip;\n uniform float uPlaneScale;\n uniform float uProjectionMix;\n - uniform float uTanHalfFovY;\n + uniform float uTanHalfSourceFovY;\n out vec2 vUv;\n + out float vEdgeMetric;\n + float shapeDepth(float rawDepth){\n + float depth = pow(max(rawDepth, 0.0), uZGamma);\n + return clamp(depth * uZScale, 0.0, uZMaxClip);\n + }\n + float relativeDiff(float a, float b){\n + return abs(a - b) / max(max(a, b), 1e-3);\n + }\n void main(){\n // Keep video座標は左→右、上→下。サンプルは上下反転のみ。 vUv = vec2(aUv.x, 1.0 - aUv.y); - float depth = texture(uDepthTex, vUv).r; - - depth = pow(max(depth, 0.0), uZGamma); + vec2 texel = vec2(1.0) / max(uDepthSize, vec2(1.0));\n + float zDepth = shapeDepth(texture(uDepthTex, vUv).r);\n + float leftDepth = shapeDepth(texture(uDepthTex, vUv + vec2(-texel.x, 0.0)).r);\n + float rightDepth = shapeDepth(texture(uDepthTex, vUv + vec2(texel.x, 0.0)).r);\n + float upDepth = shapeDepth(texture(uDepthTex, vUv + vec2(0.0, texel.y)).r);\n + float downDepth = shapeDepth(texture(uDepthTex, vUv + vec2(0.0, -texel.y)).r);\n + vEdgeMetric = max(\n + max(relativeDiff(zDepth, leftDepth), relativeDiff(zDepth, rightDepth)),\n + max(relativeDiff(zDepth, upDepth), relativeDiff(zDepth, downDepth))\n + );\n // Clip only the depth-derived component. Bias is a global offset and // should not be clamped, otherwise it saturates Z to a constant and // the mesh looks flat. - float zDepth = clamp(depth * uZScale, 0.0, uZMaxClip); float z = zDepth + uZBias; float reliefX = (aUv.x - 0.5) * uAspect * uPlaneScale;\n float reliefY = (0.5 - aUv.y) * uPlaneScale;\n - float pinholeSpread = (2.0 * uTanHalfFovY) * (0.5 * uPlaneScale);\n + float pinholeSpread = (2.0 * uTanHalfSourceFovY) * (0.5 * uPlaneScale);\n float pinholeX = (aUv.x - 0.5) * uAspect * pinholeSpread * z;\n float pinholeY = (0.5 - aUv.y) * pinholeSpread * z;\n float x = mix(reliefX, pinholeX, uProjectionMix);\n @@ -737,14 +758,20 @@ export class RawXRTest { precision highp float;\n in vec2 vUv;\n uniform sampler2D uVideoTex;\n + uniform float uEdgeDiscardThreshold;\n + in float vEdgeMetric;\n out vec4 fragColor;\n void main(){\n + if (uEdgeDiscardThreshold > 0.0 && vEdgeMetric > uEdgeDiscardThreshold) {\n + discard;\n + }\n fragColor = texture(uVideoTex, vUv);\n }`; prog = this.makeProgram(gl, vs, fs); uniforms = { viewProj: gl.getUniformLocation(prog, 'uViewProj'), model: gl.getUniformLocation(prog, 'uModel'), + depthSize: gl.getUniformLocation(prog, 'uDepthSize'), aspect: gl.getUniformLocation(prog, 'uAspect'), zScale: gl.getUniformLocation(prog, 'uZScale'), zBias: gl.getUniformLocation(prog, 'uZBias'), @@ -752,7 +779,8 @@ export class RawXRTest { zMaxClip: gl.getUniformLocation(prog, 'uZMaxClip'), planeScale: gl.getUniformLocation(prog, 'uPlaneScale'), projectionMix: gl.getUniformLocation(prog, 'uProjectionMix'), - tanHalfFovY: gl.getUniformLocation(prog, 'uTanHalfFovY'), + tanHalfSourceFovY: gl.getUniformLocation(prog, 'uTanHalfSourceFovY'), + edgeDiscardThreshold: gl.getUniformLocation(prog, 'uEdgeDiscardThreshold'), }; gl.useProgram(prog); const depthLoc = gl.getUniformLocation(prog, 'uDepthTex'); From 1636c6f0f973a32f416ba418a7f5093b673723d9 Mon Sep 17 00:00:00 2001 From: amariichi <68761912+amariichi@users.noreply.github.com> Date: Tue, 24 Mar 2026 19:50:19 +0900 Subject: [PATCH 03/11] Remove phase-2 edge cutting --- webapp/src/render/projection.ts | 4 ---- webapp/src/render/scene.ts | 7 +------ webapp/src/render/shaders.ts | 32 ++++++----------------------- webapp/src/state/playerStore.ts | 1 - webapp/src/types.ts | 1 - webapp/src/ui/controls.ts | 3 --- webapp/src/xrTest.ts | 36 ++++----------------------------- 7 files changed, 11 insertions(+), 73 deletions(-) diff --git a/webapp/src/render/projection.ts b/webapp/src/render/projection.ts index 7e11e66..f3fddab 100644 --- a/webapp/src/render/projection.ts +++ b/webapp/src/render/projection.ts @@ -7,7 +7,3 @@ export function getProjectionMix(controls: ViewerControls): number { export function getTanHalfSourceFovY(controls: ViewerControls): number { return Math.tan((controls.sourceFovY * Math.PI) / 360); } - -export function getEffectiveEdgeDiscardThreshold(controls: ViewerControls): number { - return controls.projectionMode === 'pinhole' ? controls.edgeDiscardThreshold : 0.0; -} diff --git a/webapp/src/render/scene.ts b/webapp/src/render/scene.ts index 2dd8ee1..fc6acea 100644 --- a/webapp/src/render/scene.ts +++ b/webapp/src/render/scene.ts @@ -3,7 +3,7 @@ import * as THREE from 'three'; // VRButton unused (RawXRに統一) import type { DepthFrame, ViewerControls } from '../types'; import { createGridGeometry } from './mesh'; -import { getEffectiveEdgeDiscardThreshold, getProjectionMix, getTanHalfSourceFovY } from './projection'; +import { getProjectionMix, getTanHalfSourceFovY } from './projection'; import { fragmentShader, vertexShader } from './shaders'; import { perfStats } from '../utils/perfStats'; @@ -232,7 +232,6 @@ export class RenderScene { uniforms.planeScale.value = this.currentControls.planeScale; uniforms.projectionMix.value = getProjectionMix(this.currentControls); uniforms.tanHalfSourceFovY.value = getTanHalfSourceFovY(this.currentControls); - uniforms.edgeDiscardThreshold.value = getEffectiveEdgeDiscardThreshold(this.currentControls); this.mesh.position.y = this.currentControls.yOffset; } } @@ -266,7 +265,6 @@ export class RenderScene { this.mesh.material.uniforms.depthTexture.value = this.depthTexture; } const uniforms = this.mesh.material.uniforms; - uniforms.depthSize.value.set(frame.width, frame.height); uniforms.aspect.value = frame.width / frame.height; uniforms.zScale.value = controls.zScale; uniforms.zBias.value = controls.zBias; @@ -275,7 +273,6 @@ export class RenderScene { uniforms.planeScale.value = controls.planeScale; uniforms.projectionMix.value = getProjectionMix(controls); uniforms.tanHalfSourceFovY.value = getTanHalfSourceFovY(controls); - uniforms.edgeDiscardThreshold.value = getEffectiveEdgeDiscardThreshold(controls); const yOffset = this.renderer.xr.isPresenting ? controls.yOffset + this.vrYOffset : controls.yOffset; this.mesh.position.y = yOffset; perfStats.add('scene.updateDepth', performance.now() - start); @@ -297,7 +294,6 @@ export class RenderScene { uniforms: { depthTexture: { value: this.depthTexture }, videoTexture: { value: this.videoTexture }, - depthSize: { value: new THREE.Vector2(1, 1) }, aspect: { value: aspect }, zScale: { value: controls.zScale }, zBias: { value: controls.zBias }, @@ -306,7 +302,6 @@ export class RenderScene { planeScale: { value: controls.planeScale }, projectionMix: { value: getProjectionMix(controls) }, tanHalfSourceFovY: { value: getTanHalfSourceFovY(controls) }, - edgeDiscardThreshold: { value: getEffectiveEdgeDiscardThreshold(controls) }, }, vertexShader, fragmentShader, diff --git a/webapp/src/render/shaders.ts b/webapp/src/render/shaders.ts index 14d11d2..a53b3f5 100644 --- a/webapp/src/render/shaders.ts +++ b/webapp/src/render/shaders.ts @@ -1,6 +1,5 @@ export const vertexShader = /* glsl */ ` uniform sampler2D depthTexture; - uniform vec2 depthSize; uniform float aspect; uniform float zScale; uniform float zBias; @@ -11,35 +10,21 @@ export const vertexShader = /* glsl */ ` uniform float tanHalfSourceFovY; varying vec2 vUv; varying vec2 vSampleUv; - varying float vEdgeMetric; float readDepth(vec2 uv) { return texture(depthTexture, uv).r; } - float shapeDepth(float rawDepth) { - float depth = pow(max(rawDepth, 0.0), zGamma); - return clamp(depth * zScale, 0.0, zMaxClip); - } - - float relativeDiff(float a, float b) { - return abs(a - b) / max(max(a, b), 1e-3); - } - void main() { vUv = uv; vSampleUv = vec2(1.0 - uv.x, 1.0 - uv.y); - vec2 texel = vec2(1.0) / max(depthSize, vec2(1.0)); - float zDepth = shapeDepth(readDepth(vSampleUv)); + float depth = readDepth(vSampleUv); + depth = pow(max(depth, 0.0), zGamma); + // Apply clipping to the depth-derived displacement only. + // Z Bias is a global offset and should not participate in clipping, otherwise + // non-zero bias easily saturates to a constant Z and the mesh appears flat. + float zDepth = clamp(depth * zScale, 0.0, zMaxClip); float z = zDepth + zBias; - float leftDepth = shapeDepth(readDepth(vSampleUv + vec2(-texel.x, 0.0))); - float rightDepth = shapeDepth(readDepth(vSampleUv + vec2(texel.x, 0.0))); - float upDepth = shapeDepth(readDepth(vSampleUv + vec2(0.0, texel.y))); - float downDepth = shapeDepth(readDepth(vSampleUv + vec2(0.0, -texel.y))); - vEdgeMetric = max( - max(relativeDiff(zDepth, leftDepth), relativeDiff(zDepth, rightDepth)), - max(relativeDiff(zDepth, upDepth), relativeDiff(zDepth, downDepth)) - ); float reliefX = (0.5 - vUv.x) * aspect * planeScale; float reliefY = (0.5 - vUv.y) * planeScale; // For pinhole mode, normalize the existing planeScale so the default value @@ -56,15 +41,10 @@ export const vertexShader = /* glsl */ ` export const fragmentShader = /* glsl */ ` uniform sampler2D videoTexture; - uniform float edgeDiscardThreshold; varying vec2 vUv; varying vec2 vSampleUv; - varying float vEdgeMetric; void main() { - if (edgeDiscardThreshold > 0.0 && vEdgeMetric > edgeDiscardThreshold) { - discard; - } vec4 color = texture(videoTexture, vSampleUv); gl_FragColor = color; } diff --git a/webapp/src/state/playerStore.ts b/webapp/src/state/playerStore.ts index 407c58c..26e4cba 100644 --- a/webapp/src/state/playerStore.ts +++ b/webapp/src/state/playerStore.ts @@ -26,7 +26,6 @@ const defaultControls: ViewerControls = { zBias: 0.0, zGamma: 1.0, zMaxClip: 50, - edgeDiscardThreshold: 0.2, planeScale: 2.0, yOffset: 1.2, }; diff --git a/webapp/src/types.ts b/webapp/src/types.ts index 6063b50..4bc2ba4 100644 --- a/webapp/src/types.ts +++ b/webapp/src/types.ts @@ -53,7 +53,6 @@ export interface ViewerControls { zBias: number; zGamma: number; zMaxClip: number; - edgeDiscardThreshold: number; planeScale: number; yOffset: number; } diff --git a/webapp/src/ui/controls.ts b/webapp/src/ui/controls.ts index 0b32b88..1ad6323 100644 --- a/webapp/src/ui/controls.ts +++ b/webapp/src/ui/controls.ts @@ -68,9 +68,6 @@ export class ControlPanel { folder.add(store.viewerControls, 'zMaxClip', 5, 100, 1).name('Z Max Clip').onChange((value: number) => { usePlayerStore.getState().updateControls({ zMaxClip: value }); }); - folder.add(store.viewerControls, 'edgeDiscardThreshold', 0.0, 0.6, 0.01).name('Edge Cut').onChange((value: number) => { - usePlayerStore.getState().updateControls({ edgeDiscardThreshold: value }); - }); folder.add(store.viewerControls, 'planeScale', 0.5, 3.5, 0.1).name('Plane Scale').onChange((value: number) => { usePlayerStore.getState().updateControls({ planeScale: value }); }); diff --git a/webapp/src/xrTest.ts b/webapp/src/xrTest.ts index 2f73721..a04d79e 100644 --- a/webapp/src/xrTest.ts +++ b/webapp/src/xrTest.ts @@ -3,7 +3,7 @@ import * as THREE from 'three'; import { drawThreeStereo } from './xrThreeBridge'; import type { DepthFrame, ViewerControls } from './types'; -import { getEffectiveEdgeDiscardThreshold, getProjectionMix, getTanHalfSourceFovY } from './render/projection'; +import { getProjectionMix, getTanHalfSourceFovY } from './render/projection'; import { DepthBuffer } from './utils/depthBuffer'; type RawXRMode = 'quad' | 'mesh' | 'three'; @@ -20,7 +20,6 @@ interface MeshState { uniforms: { viewProj: WebGLUniformLocation | null; model: WebGLUniformLocation | null; - depthSize: WebGLUniformLocation | null; aspect: WebGLUniformLocation | null; zScale: WebGLUniformLocation | null; zBias: WebGLUniformLocation | null; @@ -29,7 +28,6 @@ interface MeshState { planeScale: WebGLUniformLocation | null; projectionMix: WebGLUniformLocation | null; tanHalfSourceFovY: WebGLUniformLocation | null; - edgeDiscardThreshold: WebGLUniformLocation | null; }; } @@ -567,7 +565,6 @@ export class RawXRTest { gl.useProgram(s.mesh.prog); gl.uniformMatrix4fv(s.mesh.uniforms.viewProj, false, viewProj.elements as unknown as Float32List); gl.uniformMatrix4fv(s.mesh.uniforms.model, false, model.elements as unknown as Float32List); - gl.uniform2f(s.mesh.uniforms.depthSize, depth.width, depth.height); gl.uniform1f(s.mesh.uniforms.aspect, depth.width / depth.height); gl.uniform1f(s.mesh.uniforms.zScale, controls.zScale); gl.uniform1f(s.mesh.uniforms.zBias, controls.zBias); @@ -576,7 +573,6 @@ export class RawXRTest { gl.uniform1f(s.mesh.uniforms.planeScale, controls.planeScale); gl.uniform1f(s.mesh.uniforms.projectionMix, getProjectionMix(controls)); gl.uniform1f(s.mesh.uniforms.tanHalfSourceFovY, getTanHalfSourceFovY(controls)); - gl.uniform1f(s.mesh.uniforms.edgeDiscardThreshold, getEffectiveEdgeDiscardThreshold(controls)); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, s.mesh.depthTex); @@ -684,7 +680,6 @@ export class RawXRTest { zBias: 0.0, zGamma: 1.0, zMaxClip: 5.0, - edgeDiscardThreshold: 0.2, planeScale: 1.0, yOffset: 0.0, }; @@ -709,7 +704,6 @@ export class RawXRTest { uniform mat4 uViewProj;\n uniform mat4 uModel;\n uniform sampler2D uDepthTex;\n - uniform vec2 uDepthSize;\n uniform float uAspect;\n uniform float uZScale;\n uniform float uZBias;\n @@ -719,30 +713,15 @@ export class RawXRTest { uniform float uProjectionMix;\n uniform float uTanHalfSourceFovY;\n out vec2 vUv;\n - out float vEdgeMetric;\n - float shapeDepth(float rawDepth){\n - float depth = pow(max(rawDepth, 0.0), uZGamma);\n - return clamp(depth * uZScale, 0.0, uZMaxClip);\n - }\n - float relativeDiff(float a, float b){\n - return abs(a - b) / max(max(a, b), 1e-3);\n - }\n void main(){\n // Keep video座標は左→右、上→下。サンプルは上下反転のみ。 vUv = vec2(aUv.x, 1.0 - aUv.y); - vec2 texel = vec2(1.0) / max(uDepthSize, vec2(1.0));\n - float zDepth = shapeDepth(texture(uDepthTex, vUv).r);\n - float leftDepth = shapeDepth(texture(uDepthTex, vUv + vec2(-texel.x, 0.0)).r);\n - float rightDepth = shapeDepth(texture(uDepthTex, vUv + vec2(texel.x, 0.0)).r);\n - float upDepth = shapeDepth(texture(uDepthTex, vUv + vec2(0.0, texel.y)).r);\n - float downDepth = shapeDepth(texture(uDepthTex, vUv + vec2(0.0, -texel.y)).r);\n - vEdgeMetric = max(\n - max(relativeDiff(zDepth, leftDepth), relativeDiff(zDepth, rightDepth)),\n - max(relativeDiff(zDepth, upDepth), relativeDiff(zDepth, downDepth))\n - );\n + float depth = texture(uDepthTex, vUv).r;\n + depth = pow(max(depth, 0.0), uZGamma);\n // Clip only the depth-derived component. Bias is a global offset and // should not be clamped, otherwise it saturates Z to a constant and // the mesh looks flat. + float zDepth = clamp(depth * uZScale, 0.0, uZMaxClip);\n float z = zDepth + uZBias; float reliefX = (aUv.x - 0.5) * uAspect * uPlaneScale;\n float reliefY = (0.5 - aUv.y) * uPlaneScale;\n @@ -758,20 +737,14 @@ export class RawXRTest { precision highp float;\n in vec2 vUv;\n uniform sampler2D uVideoTex;\n - uniform float uEdgeDiscardThreshold;\n - in float vEdgeMetric;\n out vec4 fragColor;\n void main(){\n - if (uEdgeDiscardThreshold > 0.0 && vEdgeMetric > uEdgeDiscardThreshold) {\n - discard;\n - }\n fragColor = texture(uVideoTex, vUv);\n }`; prog = this.makeProgram(gl, vs, fs); uniforms = { viewProj: gl.getUniformLocation(prog, 'uViewProj'), model: gl.getUniformLocation(prog, 'uModel'), - depthSize: gl.getUniformLocation(prog, 'uDepthSize'), aspect: gl.getUniformLocation(prog, 'uAspect'), zScale: gl.getUniformLocation(prog, 'uZScale'), zBias: gl.getUniformLocation(prog, 'uZBias'), @@ -780,7 +753,6 @@ export class RawXRTest { planeScale: gl.getUniformLocation(prog, 'uPlaneScale'), projectionMix: gl.getUniformLocation(prog, 'uProjectionMix'), tanHalfSourceFovY: gl.getUniformLocation(prog, 'uTanHalfSourceFovY'), - edgeDiscardThreshold: gl.getUniformLocation(prog, 'uEdgeDiscardThreshold'), }; gl.useProgram(prog); const depthLoc = gl.getUniformLocation(prog, 'uDepthTex'); From e1736afe345c9c5aa973e9f27a7d28b7d1c6e305 Mon Sep 17 00:00:00 2001 From: amariichi <68761912+amariichi@users.noreply.github.com> Date: Tue, 24 Mar 2026 19:59:58 +0900 Subject: [PATCH 04/11] Add 2D camera distance control --- webapp/src/app.ts | 5 +++++ webapp/src/render/scene.ts | 27 +++++++++++++++++++++++++-- webapp/src/ui/controls.ts | 13 +++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/webapp/src/app.ts b/webapp/src/app.ts index 7b4ef26..4b2b5fe 100644 --- a/webapp/src/app.ts +++ b/webapp/src/app.ts @@ -51,8 +51,13 @@ export class VideoDepthApp { { onFileSelected: async (file) => this.handleFileSelected(file), getFrontendFps: () => this.getFrontendFps(), + getViewDistance: () => this.renderScene.getViewDistance(), + onViewDistanceChanged: (distance) => this.renderScene.setViewDistance(distance), } ); + this.renderScene.setViewDistanceChangeHandler((distance) => { + this.controlPanel.setViewDistance(distance); + }); // Manual Open Video button handler to ensure video is paused BEFORE file dialog opens const openBtn = root.querySelector('#btn-open-file'); diff --git a/webapp/src/render/scene.ts b/webapp/src/render/scene.ts index fc6acea..dec5f2f 100644 --- a/webapp/src/render/scene.ts +++ b/webapp/src/render/scene.ts @@ -47,6 +47,7 @@ export class RenderScene { ]; private rawXRViewport = new THREE.Vector4(); private manualPrevXREnabled: boolean | null = null; + private viewDistanceChangeHandler: ((distance: number) => void) | null = null; constructor( container: HTMLElement, @@ -94,6 +95,7 @@ export class RenderScene { const point = new THREE.PointLight(0xffffff, 0.8); point.position.set(2, 3, 2); this.scene.add(point); + this.orbitRadius = this.clampViewDistance(this.orbitRadius); this.camera = new THREE.PerspectiveCamera(initialControls.fovY, this.aspectRatio(), 0.01, 1000); this.camera.position.set(0, 1.4, this.orbitRadius); this.camera.lookAt(this.orbitTarget); @@ -166,6 +168,22 @@ export class RenderScene { this.handleResize(); } + public getViewDistance(): number { + return this.orbitRadius; + } + + public setViewDistance(distance: number): void { + this.orbitRadius = this.clampViewDistance(distance); + if (!this.renderer.xr.isPresenting) { + this.updateCameraOrbit(); + } + this.viewDistanceChangeHandler?.(this.orbitRadius); + } + + public setViewDistanceChangeHandler(handler: (distance: number) => void): void { + this.viewDistanceChangeHandler = handler; + } + public isSbsEnabled(): boolean { return this.sbsEnabled; } @@ -391,9 +409,9 @@ export class RenderScene { public resetView(): void { this.orbitYaw = 0; this.orbitPitch = 0; - this.orbitRadius = 3.5; this.orbitTarget.copy(this.defaultTarget); this.updateCameraOrbit(); + this.viewDistanceChangeHandler?.(this.orbitRadius); } private onXRSessionStart(): void { @@ -520,11 +538,16 @@ export class RenderScene { }); dom.addEventListener('wheel', (e) => { e.preventDefault(); - this.orbitRadius = THREE.MathUtils.clamp(this.orbitRadius + e.deltaY * 0.002, 0.6, 10); + this.orbitRadius = this.clampViewDistance(this.orbitRadius + e.deltaY * 0.002); this.updateCameraOrbit(); + this.viewDistanceChangeHandler?.(this.orbitRadius); }, { passive: false }); } + private clampViewDistance(distance: number): number { + return THREE.MathUtils.clamp(distance, 0.6, 10); + } + async startLookingGlass(): Promise { if (!this.lgPolyfillLoaded) { await this.loadLookingGlassPolyfill(); diff --git a/webapp/src/ui/controls.ts b/webapp/src/ui/controls.ts index 1ad6323..57e5137 100644 --- a/webapp/src/ui/controls.ts +++ b/webapp/src/ui/controls.ts @@ -4,6 +4,8 @@ import { usePlayerStore } from '../state/playerStore'; interface ControlCallbacks { onFileSelected: (file: File) => Promise; getFrontendFps: () => number; + getViewDistance: () => number; + onViewDistanceChanged: (distance: number) => void; } interface ControlElements { @@ -18,10 +20,13 @@ export class ControlPanel { private perfGui: GUI; private elements: ControlElements; private callbacks: ControlCallbacks; + private viewDistanceState = { cameraDistance: 3.5 }; + private cameraDistanceController: ReturnType | null = null; constructor(elements: ControlElements, callbacks: ControlCallbacks) { this.elements = elements; this.callbacks = callbacks; + this.viewDistanceState.cameraDistance = this.callbacks.getViewDistance(); this.depthGui = new GUI({ container: elements.depthGui, title: 'Depth Controls' }); this.perfGui = new GUI({ container: elements.perfGui, title: 'Performance' }); this.mountFileInput(); @@ -56,6 +61,9 @@ export class ControlPanel { folder.add(store.viewerControls, 'sourceFovY', 30, 100, 1).name('Source FOV Y').onChange((value: number) => { usePlayerStore.getState().updateControls({ sourceFovY: value }); }); + this.cameraDistanceController = folder.add(this.viewDistanceState, 'cameraDistance', 0.6, 10.0, 0.05).name('Camera Dist').onChange((value: number) => { + this.callbacks.onViewDistanceChanged(value); + }); folder.add(store.viewerControls, 'zScale', 0.5, 5.0, 0.05).name('Z Scale').onChange((value: number) => { usePlayerStore.getState().updateControls({ zScale: value }); }); @@ -148,4 +156,9 @@ export class ControlPanel { updateStatus(message: string): void { this.elements.status.textContent = message; } + + setViewDistance(distance: number): void { + this.viewDistanceState.cameraDistance = distance; + this.cameraDistanceController?.updateDisplay(); + } } From 7ea3da6d9860e149246d1f22623174a529ba8a1a Mon Sep 17 00:00:00 2001 From: amariichi <68761912+amariichi@users.noreply.github.com> Date: Tue, 24 Mar 2026 20:19:11 +0900 Subject: [PATCH 05/11] Add SBS eye swap and closer camera control --- webapp/index.html | 8 +++++++- webapp/src/app.ts | 10 +++++++++- webapp/src/render/scene.ts | 14 +++++++++++--- webapp/src/ui/controls.ts | 2 +- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/webapp/index.html b/webapp/index.html index ad92370..c1105f4 100644 --- a/webapp/index.html +++ b/webapp/index.html @@ -34,6 +34,12 @@

SBS Stereo (0DOF) + +