Support targeting morph target weights in glTF animation pointer and interactivity

Author: bghgary

packages/dev/core/src/FlowGraph/Blocks/Data/Transformers/flowGraphJsonPointerParserBlock.ts

@@ -85,10 +85,10 @@ export class FlowGraphJsonPointerParserBlock<P extends any, O extends FlowGraphA
     }
 
     public override _doOperation(context: FlowGraphContext): P {
-        const accessorContainer = this.templateComponent.getAccessor(this.config.pathConverter, context);
-        const value = accessorContainer.info.get(accessorContainer.object) as P;
-        const object = accessorContainer.info.getTarget?.(accessorContainer.object);
-        const propertyName = accessorContainer.info.getPropertyName?.[0](accessorContainer.object);
+        const accessor = this.templateComponent.getAccessor(this.config.pathConverter, context);
+        const value = accessor.info.get(accessor.object, accessor.index) as P;
+        const object = accessor.info.getTarget?.(accessor.object, accessor.index);
+        const propertyName = accessor.info.getPropertyName?.[0](accessor.object);
         if (!object) {
             throw new Error("Object is undefined");
         } else {
@@ -101,18 +101,18 @@ export class FlowGraphJsonPointerParserBlock<P extends any, O extends FlowGraphA
     }
 
     private _setPropertyValue(_target: O, _propertyName: string, value: P, context: FlowGraphContext): void {
-        const accessorContainer = this.templateComponent.getAccessor(this.config.pathConverter, context);
-        const type = accessorContainer.info.type;
+        const accessor = this.templateComponent.getAccessor(this.config.pathConverter, context);
+        const type = accessor.info.type;
         if (type.startsWith("Color")) {
             value = ToColor(value as Vector4, type) as unknown as P;
         }
-        accessorContainer.info.set?.(value, accessorContainer.object);
+        accessor.info.set?.(value, accessor.object, accessor.index);
     }
 
     private _getPropertyValue(_target: O, _propertyName: string, context: FlowGraphContext): P | undefined {
-        const accessorContainer = this.templateComponent.getAccessor(this.config.pathConverter, context);
-        const type = accessorContainer.info.type;
-        const value = accessorContainer.info.get(accessorContainer.object);
+        const accessor = this.templateComponent.getAccessor(this.config.pathConverter, context);
+        const type = accessor.info.type;
+        const value = accessor.info.get(accessor.object, accessor.index);
         if (type.startsWith("Color")) {
             return FromColor(value as Color3 | Color4) as unknown as P;
         }
@@ -124,11 +124,11 @@ export class FlowGraphJsonPointerParserBlock<P extends any, O extends FlowGraphA
         _propertyName: string,
         context: FlowGraphContext
     ): (keys: any[], fps: number, animationType: number, easingFunction?: EasingFunction) => Animation[] {
-        const accessorContainer = this.templateComponent.getAccessor(this.config.pathConverter, context);
+        const accessor = this.templateComponent.getAccessor(this.config.pathConverter, context);
         return (keys: any[], fps: number, animationType: number, easingFunction?: EasingFunction) => {
             const animations: Animation[] = [];
             // make sure keys are of the right type (in case of float3 color/vector)
-            const type = accessorContainer.info.type;
+            const type = accessor.info.type;
             if (type.startsWith("Color")) {
                 keys = keys.map((key) => {
                     return {
@@ -137,8 +137,8 @@ export class FlowGraphJsonPointerParserBlock<P extends any, O extends FlowGraphA
                     };
                 });
             }
-            accessorContainer.info.interpolation?.forEach((info, index) => {
-                const name = accessorContainer.info.getPropertyName?.[index](accessorContainer.object) || "Animation-interpolation-" + index;
+            accessor.info.interpolation?.forEach((info, index) => {
+                const name = accessor.info.getPropertyName?.[index](accessor.object) || "Animation-interpolation-" + index;
                 // generate the keys based on interpolation info
                 let newKeys: any[] = keys;
                 if (animationType !== info.type) {
@@ -150,7 +150,7 @@ export class FlowGraphJsonPointerParserBlock<P extends any, O extends FlowGraphA
                         };
                     });
                 }
-                const animationData = info.buildAnimations(accessorContainer.object, name, 60, newKeys);
+                const animationData = info.buildAnimations(accessor.object, name, 60, newKeys);
                 for (const animation of animationData) {
                     if (easingFunction) {
                         animation.babylonAnimation.setEasingFunction(easingFunction);

packages/dev/core/src/ObjectModel/objectModelInterfaces.ts

@@ -1,12 +1,15 @@
 /**
  * A container with an original object and information about that object.
- * on some other object.
  */
 export interface IObjectInfo<T, O = any> {
     /**
      * The original object.
      */
     object: O;
+    /**
+     * The index of the array in question when applicable.
+     */
+    index?: number;
     /**
      * Information about the object.
      */

packages/dev/loaders/src/glTF/2.0/Extensions/KHR_interactivity.ts

@@ -83,7 +83,7 @@ export class KHR_interactivity implements IGLTFLoaderExtension {
             return parser.serializeToFlowGraph();
         });
         // parse each graph async
-        await Promise.all(graphs.map(async (graph) => await ParseFlowGraphAsync(graph, { coordinator, pathConverter: this._pathConverter })));
+        await Promise.all(graphs.map((graph) => ParseFlowGraphAsync(graph, { coordinator, pathConverter: this._pathConverter })));
 
         coordinator.start();
     }

packages/dev/loaders/src/glTF/2.0/Extensions/gltfPathToObjectConverter.ts

@@ -2,20 +2,6 @@ import type { IObjectInfo, IPathToObjectConverter } from "core/ObjectModel/objec
 import type { IGLTF } from "../glTFLoaderInterfaces";
 import type { IObjectAccessor } from "core/FlowGraph/typeDefinitions";
 
-/**
- * Adding an exception here will break traversing through the glTF object tree.
- * This is used for properties that might not be in the glTF object model, but are optional and have a default value.
- * For example, the path /nodes/\{\}/extensions/KHR_node_visibility/visible is optional - the object can be deferred without the object fully existing.
- */
-export const OptionalPathExceptionsList: {
-    regex: RegExp;
-}[] = [
-    {
-        // get the node as object when reading an extension
-        regex: new RegExp(`^/nodes/\\d+/extensions/`),
-    },
-];
-
 /**
  * A converter that takes a glTF Object Model JSON Pointer
  * and transforms it into an ObjectAccessorContainer, allowing
@@ -42,7 +28,7 @@ export class GLTFPathToObjectConverter<T, BabylonType, BabylonValue> implements
      *
      * Examples:
      *  - "/nodes/0/rotation"
-     * - "/nodes.length"
+     *  - "/nodes.length"
      *  - "/materials/2/emissiveFactor"
      *  - "/materials/2/pbrMetallicRoughness/baseColorFactor"
      *  - "/materials/2/extensions/KHR_materials_emissive_strength/emissiveStrength"
@@ -61,51 +47,40 @@ export class GLTFPathToObjectConverter<T, BabylonType, BabylonValue> implements
         const parts = path.split("/");
         parts.shift();
 
-        //if the last part has ".length" in it, separate that as an extra part
-        if (parts[parts.length - 1].includes(".length")) {
+        // If the last part ends with `.length`, separate that as an extra part
+        if (parts[parts.length - 1].endsWith(".length")) {
             const lastPart = parts[parts.length - 1];
             const split = lastPart.split(".");
             parts.pop();
             parts.push(...split);
         }
 
-        let ignoreObjectTree = false;
+        let arrayIndex: number | undefined = undefined;
 
         for (const part of parts) {
-            const isLength = part === "length";
-            if (isLength && !infoTree.__array__) {
-                throw new Error(`Path ${path} is invalid`);
-            }
-            if (infoTree.__ignoreObjectTree__) {
-                ignoreObjectTree = true;
-            }
-            if (infoTree.__array__ && !isLength) {
-                infoTree = infoTree.__array__;
-            } else {
+            if (infoTree[part]) {
                 infoTree = infoTree[part];
-                if (!infoTree) {
-                    throw new Error(`Path ${path} is invalid`);
+            } else if (infoTree.__array__) {
+                infoTree = infoTree.__array__;
+                if (target) {
+                    arrayIndex = parseInt(part, 10);
                 }
+            } else {
+                throw new Error(`Path ${path} is invalid`);
             }
-            if (!ignoreObjectTree) {
-                if (objectTree === undefined) {
-                    // check if the path is in the exception list. If it is, break and return the last object that was found
-                    const exception = OptionalPathExceptionsList.find((e) => e.regex.test(path));
-                    if (!exception) {
-                        throw new Error(`Path ${path} is invalid`);
-                    }
-                } else if (!isLength) {
-                    objectTree = objectTree?.[part];
-                }
+
+            if (!target) {
+                objectTree = objectTree[part];
             }
 
-            if (infoTree.__target__ || isLength) {
+            if (infoTree.__target__) {
                 target = objectTree;
             }
         }
 
         return {
             object: target,
+            index: arrayIndex,
             info: infoTree,
         };
     }

packages/dev/loaders/src/glTF/2.0/Extensions/objectModelMapping.ts

@@ -44,7 +44,7 @@ export interface IGLTFObjectModelTreeNodesObject<GLTFTargetType = INode, Babylon
         globalMatrix: IObjectAccessor<GLTFTargetType, BabylonTargetType, Matrix>;
         weights: {
             length: IObjectAccessor<GLTFTargetType, BabylonTargetType, number>;
-            __array__: { __target__: boolean } & IObjectAccessor<GLTFTargetType, BabylonTargetType, number>;
+            __array__: IObjectAccessor<GLTFTargetType, BabylonTargetType, number>;
         } & IObjectAccessor<GLTFTargetType, BabylonTargetType, number[]>;
         extensions: {
             EXT_lights_ies?: {
@@ -304,21 +304,33 @@ const nodesTree: IGLTFObjectModelTreeNodesObject = {
                 type: "number",
                 get: (node: INode) => node._numMorphTargets,
                 getTarget: (node: INode) => node._babylonTransformNode,
-                getPropertyName: [() => "influence"],
             },
             __array__: {
-                __target__: true,
                 type: "number",
-                get: (node: INode, index?: number) => (index !== undefined ? node._primitiveBabylonMeshes?.[0].morphTargetManager?.getTarget(index).influence : undefined),
-                // set: (value: number, node: INode, index?: number) => node._babylonTransformNode?.getMorphTargetManager()?.getTarget(index)?.setInfluence(value),
+                get: (node: INode, arrayIndex?: number) => node._primitiveBabylonMeshes![0].morphTargetManager!.getTarget(arrayIndex!).influence,
+                set: (value: number, node: INode, arrayIndex?: number) => {
+                    for (const primitive of node._primitiveBabylonMeshes!) {
+                        const target = primitive.morphTargetManager!.getTarget(arrayIndex!);
+                        if (target) {
+                            target.influence = value;
+                        }
+                    }
+                },
                 getTarget: (node: INode) => node._babylonTransformNode,
-                getPropertyName: [() => "influence"],
             },
             type: "number[]",
-            get: (node: INode, index?: number) => [0], // TODO: get the weights correctly
-            // set: (value: number, node: INode, index?: number) => node._babylonTransformNode?.getMorphTargetManager()?.getTarget(index)?.setInfluence(value),
+            get: (node: INode) => Array.from({ length: node._numMorphTargets! }, (_, index) => node._primitiveBabylonMeshes![0].morphTargetManager!.getTarget(index).influence!),
+            set: (value: number[], node: INode) => {
+                for (const primitive of node._primitiveBabylonMeshes!) {
+                    for (let i = 0; i < value.length; i++) {
+                        const target = primitive.morphTargetManager!.getTarget(i);
+                        if (target) {
+                            target.influence = value[i];
+                        }
+                    }
+                }
+            },
             getTarget: (node: INode) => node._babylonTransformNode,
-            getPropertyName: [() => "influence"],
         },
         // readonly!
         matrix: {

packages/public/glTF2Interface/babylon.glTF2Interface.d.ts

@@ -691,7 +691,7 @@ declare module BABYLON.GLTF2 {
          */
         mode?: MeshPrimitiveMode;
         /**
-         * An array of Morph Targets, each  Morph Target is a dictionary mapping attributes (only POSITION, NORMAL, and TANGENT supported) to their deviations in the Morph Target
+         * An array of morph targets, each morph target is a dictionary mapping attributes (only POSITION, NORMAL, and TANGENT supported) to their deviations in the morph target
          */
         targets?: {
             [name: string]: number;