Feat/depth mesh normview transform

src/depth/Depth.test.ts

@@ -0,0 +1,146 @@
+import {describe, it, expect, vi} from 'vitest';
+import * as THREE from 'three';
+
+import {Depth} from './Depth';
+
+describe('Depth', () => {
+  /**
+   * Creates a fresh Depth instance, clearing the singleton so each test
+   * gets its own state.
+   */
+  function createDepth(): Depth {
+    // Reset singleton between tests.
+    Depth.instance = undefined;
+    return new Depth();
+  }
+
+  describe('getDepth with normDepthBufferFromNormView', () => {
+    it('returns 0 when no depth data is available', () => {
+      const depth = createDepth();
+      expect(depth.getDepth(0.5, 0.5)).toBe(0);
+    });
+
+    it('reads depth without transform when matrix array is empty', () => {
+      const depth = createDepth();
+      // Set up a 2x2 depth buffer with known values.
+      depth.width = 2;
+      depth.height = 2;
+      // Row-major: [top-left, top-right, bottom-left, bottom-right]
+      // Note: getDepth uses (1-v) for Y, so v=0 maps to last row.
+      depth.depthArray[0] = new Float32Array([10, 20, 30, 40]);
+      depth.cpuDepthData[0] = {rawValueToMeters: 0.1} as XRCPUDepthInformation;
+
+      // u=0, v=1 should map to depthX=0, depthY=0 -> value 10
+      expect(depth.getDepth(0, 1)).toBeCloseTo(1.0);
+      // u=1, v=0 should map to depthX=1 (round(1*1)=1), depthY=1 -> value 40
+      // Actually: depthX = round(1*2) clamped to 1, depthY = round(1*2) clamped to 1
+      // -> index = 1*2+1 = 3 -> value 40
+      expect(depth.getDepth(1, 0)).toBeCloseTo(4.0);
+    });
+
+    it('applies normDepthBufferFromNormView transform to coordinates', () => {
+      const depth = createDepth();
+      depth.width = 2;
+      depth.height = 2;
+      depth.depthArray[0] = new Float32Array([10, 20, 30, 40]);
+      depth.cpuDepthData[0] = {rawValueToMeters: 0.1} as XRCPUDepthInformation;
+
+      // Set up a transform that swaps u and v (simulating a 90-degree rotation
+      // between view and depth buffer coordinate systems).
+      const swapMatrix = new THREE.Matrix4().set(
+        0,
+        1,
+        0,
+        0, // new_x = old_y
+        1,
+        0,
+        0,
+        0, // new_y = old_x
+        0,
+        0,
+        1,
+        0,
+        0,
+        0,
+        0,
+        1
+      );
+      depth.normDepthBufferFromNormViewMatrices[0] = swapMatrix;
+
+      // Without the transform, getDepth(0, 1) would read (u=0, v=1).
+      // With the swap, it becomes (u=1, v=0), reading the opposite corner.
+      const withTransform = depth.getDepth(0, 1);
+      // u=1,v=0 -> depthX=round(1*2) clamped 1, depthY=round(1*2) clamped 1
+      // -> index 3 -> value 40 * 0.1 = 4.0
+      expect(withTransform).toBeCloseTo(4.0);
+    });
+  });
+
+  describe('getVertex with normDepthBufferFromNormView', () => {
+    it('returns null when no depth data is available', () => {
+      const depth = createDepth();
+      expect(depth.getVertex(0.5, 0.5)).toBeNull();
+    });
+
+    it('applies transform before looking up depth', () => {
+      const depth = createDepth();
+      depth.width = 2;
+      depth.height = 2;
+      depth.depthArray[0] = new Float32Array([10, 20, 30, 40]);
+      depth.cpuDepthData[0] = {rawValueToMeters: 0.1} as XRCPUDepthInformation;
+      depth.depthProjectionInverseMatrices[0] = new THREE.Matrix4(); // identity
+
+      // Identity transform — result should be the same as no transform.
+      depth.normDepthBufferFromNormViewMatrices[0] = new THREE.Matrix4();
+      const vertex = depth.getVertex(0, 1);
+      expect(vertex).not.toBeNull();
+    });
+  });
+
+  describe('shouldUpdateDepthMesh throttling', () => {
+    it('always updates when depthMeshUpdateFps is 0', () => {
+      const depth = createDepth();
+      depth.options.depthMesh.depthMeshUpdateFps = 0;
+
+      // Access the private method via bracket notation.
+      const shouldUpdate = (depth as unknown as Record<string, () => boolean>)[
+        'shouldUpdateDepthMesh'
+      ];
+      expect(shouldUpdate.call(depth)).toBe(true);
+      expect(shouldUpdate.call(depth)).toBe(true);
+      expect(shouldUpdate.call(depth)).toBe(true);
+    });
+
+    it('throttles updates when depthMeshUpdateFps is set', () => {
+      const depth = createDepth();
+      depth.options.depthMesh.depthMeshUpdateFps = 10; // 100ms between updates
+
+      const shouldUpdate = (depth as unknown as Record<string, () => boolean>)[
+        'shouldUpdateDepthMesh'
+      ];
+
+      // First call should always succeed.
+      expect(shouldUpdate.call(depth)).toBe(true);
+
+      // Immediate second call should be throttled.
+      expect(shouldUpdate.call(depth)).toBe(false);
+    });
+
+    it('allows update after enough time has passed', () => {
+      const depth = createDepth();
+      depth.options.depthMesh.depthMeshUpdateFps = 10; // 100ms interval
+
+      const shouldUpdate = (depth as unknown as Record<string, () => boolean>)[
+        'shouldUpdateDepthMesh'
+      ];
+
+      expect(shouldUpdate.call(depth)).toBe(true);
+
+      // Fast-forward time by 150ms.
+      vi.spyOn(performance, 'now').mockReturnValue(performance.now() + 150);
+      expect(shouldUpdate.call(depth)).toBe(true);
+
+      vi.restoreAllMocks();
+    });
+  });
+});

src/depth/Depth.ts

@@ -12,6 +12,7 @@ import {OcclusionPass} from './occlusion/OcclusionPass';
 const DEFAULT_DEPTH_WIDTH = 160;
 const DEFAULT_DEPTH_HEIGHT = DEFAULT_DEPTH_WIDTH;
 const clipSpacePosition = new THREE.Vector3();
+const normViewCoord = new THREE.Vector3();
 
 export type DepthArray = Float32Array | Uint16Array;
 
@@ -54,6 +55,15 @@ export class Depth {
   depthCameraPositions: THREE.Vector3[] = [];
   depthCameraRotations: THREE.Quaternion[] = [];
 
+  /**
+   * Transforms from normalized view coordinates to normalized depth buffer
+   * coordinates. Identity when matchDepthView is true.
+   */
+  normDepthBufferFromNormViewMatrices: THREE.Matrix4[] = [];
+
+  /** Timestamp of the last depth mesh geometry update. */
+  private lastDepthMeshUpdateTime = 0;
+
   /**
    * Depth is a lightweight manager based on three.js to simply prototyping
    * with Depth in WebXR.
@@ -107,12 +117,21 @@ export class Depth {
 
   /**
    * Retrieves the depth at normalized coordinates (u, v).
+   * Note: The UV coordinates are with respect to the user's view, not the depth camera view.
    * @param u - Normalized horizontal coordinate.
    * @param v - Normalized vertical coordinate.
    * @returns Depth value at the specified coordinates.
    */
   getDepth(u: number, v: number) {
     if (!this.depthArray[0]) return 0.0;
+    // When matchDepthView is false, transform from view-space UVs to
+    // depth buffer UVs using normDepthBufferFromNormView.
+    if (this.normDepthBufferFromNormViewMatrices.length > 0) {
+      normViewCoord.set(u, v, 0);
+      normViewCoord.applyMatrix4(this.normDepthBufferFromNormViewMatrices[0]);
+      u = normViewCoord.x;
+      v = normViewCoord.y;
+    }
     const depthX = Math.round(clamp(u * this.width, 0, this.width - 1));
     const depthY = Math.round(
       clamp((1.0 - v) * this.height, 0, this.height - 1)
@@ -137,7 +156,17 @@ export class Depth {
       .applyMatrix4(this.depthProjectionMatrices[0]);
     const u = 0.5 * (clipSpacePosition.x + 1.0);
     const v = 0.5 * (clipSpacePosition.y + 1.0);
-    const depth = this.getDepth(u, v);
+
+    let depth = 0.0;
+    if (this.depthArray[0]) {
+      const depthX = Math.round(clamp(u * this.width, 0, this.width - 1));
+      const depthY = Math.round(
+        clamp((1.0 - v) * this.height, 0, this.height - 1)
+      );
+      const rawDepth = this.depthArray[0][depthY * this.width + depthX];
+      depth = this.rawValueToMeters * rawDepth;
+    }
+
     target.set(2.0 * (u - 0.5), 2.0 * (v - 0.5), -1);
     target.applyMatrix4(this.depthProjectionInverseMatrices[0]);
     target.multiplyScalar(-depth / target.z);
@@ -146,13 +175,23 @@ export class Depth {
 
   /**
    * Retrieves the depth at normalized coordinates (u, v).
+   * Note: The UV coordinates are with respect to the user's view, not the depth camera view.
    * @param u - Normalized horizontal coordinate.
    * @param v - Normalized vertical coordinate.
    * @returns Vertex at (u, v)
    */
   getVertex(u: number, v: number) {
     if (!this.depthArray[0]) return null;
 
+    // When matchDepthView is false, transform from view-space UVs to
+    // depth buffer UVs using normDepthBufferFromNormView.
+    if (this.normDepthBufferFromNormViewMatrices.length > 0) {
+      normViewCoord.set(u, v, 0);
+      normViewCoord.applyMatrix4(this.normDepthBufferFromNormViewMatrices[0]);
+      u = normViewCoord.x;
+      v = normViewCoord.y;
+    }
+
     const depthX = Math.round(clamp(u * this.width, 0, this.width - 1));
     const depthY = Math.round(
       clamp((1.0 - v) * this.height, 0, this.height - 1)
@@ -178,6 +217,15 @@ export class Depth {
       this.depthProjectionInverseMatrices.push(new THREE.Matrix4());
       this.depthCameraPositions.push(new THREE.Vector3());
       this.depthCameraRotations.push(new THREE.Quaternion());
+      this.normDepthBufferFromNormViewMatrices.push(new THREE.Matrix4());
+    }
+    // Store the view-to-depth-buffer coordinate transform.
+    if (depthData.normDepthBufferFromNormView) {
+      this.normDepthBufferFromNormViewMatrices[viewId].fromArray(
+        depthData.normDepthBufferFromNormView.matrix
+      );
+    } else {
+      this.normDepthBufferFromNormViewMatrices[viewId].identity();
     }
     if (depthData.projectionMatrix && depthData.transform) {
       this.depthProjectionMatrices[viewId].fromArray(
@@ -231,10 +279,12 @@ export class Depth {
     }
 
     if (this.options.depthMesh.enabled && this.depthMesh && viewId == 0) {
-      this.depthMesh.updateDepth(
-        depthData,
-        this.depthProjectionInverseMatrices[0]
-      );
+      if (this.shouldUpdateDepthMesh()) {
+        this.depthMesh.updateDepth(
+          depthData,
+          this.depthProjectionInverseMatrices[0]
+        );
+      }
       this.depthMesh.updatePose(
         this.depthCameraPositions[0],
         this.depthCameraRotations[0]
@@ -276,16 +326,18 @@ export class Depth {
     }
 
     if (this.options.depthMesh.enabled && this.depthMesh && viewId == 0) {
-      if (cpuDepth) {
-        this.depthMesh.updateDepth(
-          cpuDepth,
-          this.depthProjectionInverseMatrices[0]
-        );
-      } else {
-        this.depthMesh.updateGPUDepth(
-          depthData,
-          this.depthProjectionInverseMatrices[0]
-        );
+      if (this.shouldUpdateDepthMesh()) {
+        if (cpuDepth) {
+          this.depthMesh.updateDepth(
+            cpuDepth,
+            this.depthProjectionInverseMatrices[0]
+          );
+        } else {
+          this.depthMesh.updateGPUDepth(
+            depthData,
+            this.depthProjectionInverseMatrices[0]
+          );
+        }
       }
       this.depthMesh.updatePose(
         this.depthCameraPositions[0],
@@ -294,6 +346,23 @@ export class Depth {
     }
   }
 
+  /**
+   * Checks whether the depth mesh geometry should be updated this frame,
+   * based on the configured depthMeshUpdateFps. The pose is always updated
+   * every frame so the mesh tracks the depth camera smoothly, but the
+   * expensive geometry rebuild can be throttled.
+   */
+  private shouldUpdateDepthMesh(): boolean {
+    const fps = this.options.depthMesh.depthMeshUpdateFps;
+    if (fps <= 0) return true;
+    const now = performance.now();
+    if (now - this.lastDepthMeshUpdateTime < 1000 / fps) {
+      return false;
+    }
+    this.lastDepthMeshUpdateTime = now;
+    return true;
+  }
+
   getTexture(viewId: number) {
     if (!this.options.depthTexture.enabled) return undefined;
     return this.depthTextures?.get(viewId);

src/depth/DepthOptions.ts

@@ -18,6 +18,8 @@ export class DepthMeshOptions {
   // Whether to always update the full resolution geometry.
   updateFullResolutionGeometry = false;
   colliderUpdateFps = 5;
+  /** FPS cap for depth mesh geometry updates. 0 = update every frame. */
+  depthMeshUpdateFps = 0;
   depthFullResolution = 160;
   ignoreEdgePixels = 3;
 }