diff --git a/src/depth/Depth.test.ts b/src/depth/Depth.test.ts new file mode 100644 index 00000000..31e4ec08 --- /dev/null +++ b/src/depth/Depth.test.ts @@ -0,0 +1,146 @@ +import {describe, it, expect, vi} from 'vitest'; +import * as THREE from 'three'; + +import {Depth} from './Depth'; + +describe('Depth', () => { + /** + * Creates a fresh Depth instance, clearing the singleton so each test + * gets its own state. + */ + function createDepth(): Depth { + // Reset singleton between tests. + Depth.instance = undefined; + return new Depth(); + } + + describe('getDepth with normDepthBufferFromNormView', () => { + it('returns 0 when no depth data is available', () => { + const depth = createDepth(); + expect(depth.getDepth(0.5, 0.5)).toBe(0); + }); + + it('reads depth without transform when matrix array is empty', () => { + const depth = createDepth(); + // Set up a 2x2 depth buffer with known values. + depth.width = 2; + depth.height = 2; + // Row-major: [top-left, top-right, bottom-left, bottom-right] + // Note: getDepth uses (1-v) for Y, so v=0 maps to last row. + depth.depthArray[0] = new Float32Array([10, 20, 30, 40]); + depth.cpuDepthData[0] = {rawValueToMeters: 0.1} as XRCPUDepthInformation; + + // u=0, v=1 should map to depthX=0, depthY=0 -> value 10 + expect(depth.getDepth(0, 1)).toBeCloseTo(1.0); + // u=1, v=0 should map to depthX=1 (round(1*1)=1), depthY=1 -> value 40 + // Actually: depthX = round(1*2) clamped to 1, depthY = round(1*2) clamped to 1 + // -> index = 1*2+1 = 3 -> value 40 + expect(depth.getDepth(1, 0)).toBeCloseTo(4.0); + }); + + it('applies normDepthBufferFromNormView transform to coordinates', () => { + const depth = createDepth(); + depth.width = 2; + depth.height = 2; + depth.depthArray[0] = new Float32Array([10, 20, 30, 40]); + depth.cpuDepthData[0] = {rawValueToMeters: 0.1} as XRCPUDepthInformation; + + // Set up a transform that swaps u and v (simulating a 90-degree rotation + // between view and depth buffer coordinate systems). + const swapMatrix = new THREE.Matrix4().set( + 0, + 1, + 0, + 0, // new_x = old_y + 1, + 0, + 0, + 0, // new_y = old_x + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 1 + ); + depth.normDepthBufferFromNormViewMatrices[0] = swapMatrix; + + // Without the transform, getDepth(0, 1) would read (u=0, v=1). + // With the swap, it becomes (u=1, v=0), reading the opposite corner. + const withTransform = depth.getDepth(0, 1); + // u=1,v=0 -> depthX=round(1*2) clamped 1, depthY=round(1*2) clamped 1 + // -> index 3 -> value 40 * 0.1 = 4.0 + expect(withTransform).toBeCloseTo(4.0); + }); + }); + + describe('getVertex with normDepthBufferFromNormView', () => { + it('returns null when no depth data is available', () => { + const depth = createDepth(); + expect(depth.getVertex(0.5, 0.5)).toBeNull(); + }); + + it('applies transform before looking up depth', () => { + const depth = createDepth(); + depth.width = 2; + depth.height = 2; + depth.depthArray[0] = new Float32Array([10, 20, 30, 40]); + depth.cpuDepthData[0] = {rawValueToMeters: 0.1} as XRCPUDepthInformation; + depth.depthProjectionInverseMatrices[0] = new THREE.Matrix4(); // identity + + // Identity transform — result should be the same as no transform. + depth.normDepthBufferFromNormViewMatrices[0] = new THREE.Matrix4(); + const vertex = depth.getVertex(0, 1); + expect(vertex).not.toBeNull(); + }); + }); + + describe('shouldUpdateDepthMesh throttling', () => { + it('always updates when depthMeshUpdateFps is 0', () => { + const depth = createDepth(); + depth.options.depthMesh.depthMeshUpdateFps = 0; + + // Access the private method via bracket notation. + const shouldUpdate = (depth as unknown as Record boolean>)[ + 'shouldUpdateDepthMesh' + ]; + expect(shouldUpdate.call(depth)).toBe(true); + expect(shouldUpdate.call(depth)).toBe(true); + expect(shouldUpdate.call(depth)).toBe(true); + }); + + it('throttles updates when depthMeshUpdateFps is set', () => { + const depth = createDepth(); + depth.options.depthMesh.depthMeshUpdateFps = 10; // 100ms between updates + + const shouldUpdate = (depth as unknown as Record boolean>)[ + 'shouldUpdateDepthMesh' + ]; + + // First call should always succeed. + expect(shouldUpdate.call(depth)).toBe(true); + + // Immediate second call should be throttled. + expect(shouldUpdate.call(depth)).toBe(false); + }); + + it('allows update after enough time has passed', () => { + const depth = createDepth(); + depth.options.depthMesh.depthMeshUpdateFps = 10; // 100ms interval + + const shouldUpdate = (depth as unknown as Record boolean>)[ + 'shouldUpdateDepthMesh' + ]; + + expect(shouldUpdate.call(depth)).toBe(true); + + // Fast-forward time by 150ms. + vi.spyOn(performance, 'now').mockReturnValue(performance.now() + 150); + expect(shouldUpdate.call(depth)).toBe(true); + + vi.restoreAllMocks(); + }); + }); +}); diff --git a/src/depth/Depth.ts b/src/depth/Depth.ts index 8318445e..83137e98 100644 --- a/src/depth/Depth.ts +++ b/src/depth/Depth.ts @@ -12,6 +12,7 @@ import {OcclusionPass} from './occlusion/OcclusionPass'; const DEFAULT_DEPTH_WIDTH = 160; const DEFAULT_DEPTH_HEIGHT = DEFAULT_DEPTH_WIDTH; const clipSpacePosition = new THREE.Vector3(); +const normViewCoord = new THREE.Vector3(); export type DepthArray = Float32Array | Uint16Array; @@ -54,6 +55,15 @@ export class Depth { depthCameraPositions: THREE.Vector3[] = []; depthCameraRotations: THREE.Quaternion[] = []; + /** + * Transforms from normalized view coordinates to normalized depth buffer + * coordinates. Identity when matchDepthView is true. + */ + normDepthBufferFromNormViewMatrices: THREE.Matrix4[] = []; + + /** Timestamp of the last depth mesh geometry update. */ + private lastDepthMeshUpdateTime = 0; + /** * Depth is a lightweight manager based on three.js to simply prototyping * with Depth in WebXR. @@ -107,12 +117,21 @@ export class Depth { /** * Retrieves the depth at normalized coordinates (u, v). + * Note: The UV coordinates are with respect to the user's view, not the depth camera view. * @param u - Normalized horizontal coordinate. * @param v - Normalized vertical coordinate. * @returns Depth value at the specified coordinates. */ getDepth(u: number, v: number) { if (!this.depthArray[0]) return 0.0; + // When matchDepthView is false, transform from view-space UVs to + // depth buffer UVs using normDepthBufferFromNormView. + if (this.normDepthBufferFromNormViewMatrices.length > 0) { + normViewCoord.set(u, v, 0); + normViewCoord.applyMatrix4(this.normDepthBufferFromNormViewMatrices[0]); + u = normViewCoord.x; + v = normViewCoord.y; + } const depthX = Math.round(clamp(u * this.width, 0, this.width - 1)); const depthY = Math.round( clamp((1.0 - v) * this.height, 0, this.height - 1) @@ -137,7 +156,17 @@ export class Depth { .applyMatrix4(this.depthProjectionMatrices[0]); const u = 0.5 * (clipSpacePosition.x + 1.0); const v = 0.5 * (clipSpacePosition.y + 1.0); - const depth = this.getDepth(u, v); + + let depth = 0.0; + if (this.depthArray[0]) { + const depthX = Math.round(clamp(u * this.width, 0, this.width - 1)); + const depthY = Math.round( + clamp((1.0 - v) * this.height, 0, this.height - 1) + ); + const rawDepth = this.depthArray[0][depthY * this.width + depthX]; + depth = this.rawValueToMeters * rawDepth; + } + target.set(2.0 * (u - 0.5), 2.0 * (v - 0.5), -1); target.applyMatrix4(this.depthProjectionInverseMatrices[0]); target.multiplyScalar(-depth / target.z); @@ -146,6 +175,7 @@ export class Depth { /** * Retrieves the depth at normalized coordinates (u, v). + * Note: The UV coordinates are with respect to the user's view, not the depth camera view. * @param u - Normalized horizontal coordinate. * @param v - Normalized vertical coordinate. * @returns Vertex at (u, v) @@ -153,6 +183,15 @@ export class Depth { getVertex(u: number, v: number) { if (!this.depthArray[0]) return null; + // When matchDepthView is false, transform from view-space UVs to + // depth buffer UVs using normDepthBufferFromNormView. + if (this.normDepthBufferFromNormViewMatrices.length > 0) { + normViewCoord.set(u, v, 0); + normViewCoord.applyMatrix4(this.normDepthBufferFromNormViewMatrices[0]); + u = normViewCoord.x; + v = normViewCoord.y; + } + const depthX = Math.round(clamp(u * this.width, 0, this.width - 1)); const depthY = Math.round( clamp((1.0 - v) * this.height, 0, this.height - 1) @@ -178,6 +217,15 @@ export class Depth { this.depthProjectionInverseMatrices.push(new THREE.Matrix4()); this.depthCameraPositions.push(new THREE.Vector3()); this.depthCameraRotations.push(new THREE.Quaternion()); + this.normDepthBufferFromNormViewMatrices.push(new THREE.Matrix4()); + } + // Store the view-to-depth-buffer coordinate transform. + if (depthData.normDepthBufferFromNormView) { + this.normDepthBufferFromNormViewMatrices[viewId].fromArray( + depthData.normDepthBufferFromNormView.matrix + ); + } else { + this.normDepthBufferFromNormViewMatrices[viewId].identity(); } if (depthData.projectionMatrix && depthData.transform) { this.depthProjectionMatrices[viewId].fromArray( @@ -231,10 +279,12 @@ export class Depth { } if (this.options.depthMesh.enabled && this.depthMesh && viewId == 0) { - this.depthMesh.updateDepth( - depthData, - this.depthProjectionInverseMatrices[0] - ); + if (this.shouldUpdateDepthMesh()) { + this.depthMesh.updateDepth( + depthData, + this.depthProjectionInverseMatrices[0] + ); + } this.depthMesh.updatePose( this.depthCameraPositions[0], this.depthCameraRotations[0] @@ -276,16 +326,18 @@ export class Depth { } if (this.options.depthMesh.enabled && this.depthMesh && viewId == 0) { - if (cpuDepth) { - this.depthMesh.updateDepth( - cpuDepth, - this.depthProjectionInverseMatrices[0] - ); - } else { - this.depthMesh.updateGPUDepth( - depthData, - this.depthProjectionInverseMatrices[0] - ); + if (this.shouldUpdateDepthMesh()) { + if (cpuDepth) { + this.depthMesh.updateDepth( + cpuDepth, + this.depthProjectionInverseMatrices[0] + ); + } else { + this.depthMesh.updateGPUDepth( + depthData, + this.depthProjectionInverseMatrices[0] + ); + } } this.depthMesh.updatePose( this.depthCameraPositions[0], @@ -294,6 +346,23 @@ export class Depth { } } + /** + * Checks whether the depth mesh geometry should be updated this frame, + * based on the configured depthMeshUpdateFps. The pose is always updated + * every frame so the mesh tracks the depth camera smoothly, but the + * expensive geometry rebuild can be throttled. + */ + private shouldUpdateDepthMesh(): boolean { + const fps = this.options.depthMesh.depthMeshUpdateFps; + if (fps <= 0) return true; + const now = performance.now(); + if (now - this.lastDepthMeshUpdateTime < 1000 / fps) { + return false; + } + this.lastDepthMeshUpdateTime = now; + return true; + } + getTexture(viewId: number) { if (!this.options.depthTexture.enabled) return undefined; return this.depthTextures?.get(viewId); diff --git a/src/depth/DepthOptions.ts b/src/depth/DepthOptions.ts index 2f164544..4f142332 100644 --- a/src/depth/DepthOptions.ts +++ b/src/depth/DepthOptions.ts @@ -18,6 +18,8 @@ export class DepthMeshOptions { // Whether to always update the full resolution geometry. updateFullResolutionGeometry = false; colliderUpdateFps = 5; + /** FPS cap for depth mesh geometry updates. 0 = update every frame. */ + depthMeshUpdateFps = 0; depthFullResolution = 160; ignoreEdgePixels = 3; }