From f3ceb7c47c0171473c8e1247e2095d9d5ddb8236 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sat, 27 Jul 2024 12:09:13 +1000 Subject: [PATCH 01/47] gltf loader 2.0 can load geometries --- .../src/assets/loaders/gltf/GLTFParser.ts | 4 +- packages/engine/src/gltf/GLTFComponent.tsx | 5 +- packages/engine/src/gltf/GLTFDocumentState.ts | 10 +- .../engine/src/gltf/GLTFLoaderFunctions.ts | 265 +++++++++++++++++ packages/engine/src/gltf/GLTFState.tsx | 266 ++++++++++++++---- 5 files changed, 494 insertions(+), 56 deletions(-) create mode 100644 packages/engine/src/gltf/GLTFLoaderFunctions.ts diff --git a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts index e0f480ffa0..074eb1be57 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts @@ -123,7 +123,7 @@ declare module '@gltf-transform/core/dist/types/gltf.d.ts' { } } -type GLTFParserOptions = { +export type GLTFParserOptions = { crossOrigin: 'anonymous' | string ktx2Loader: KTX2Loader manager: LoadingManager | any @@ -1767,7 +1767,7 @@ export class GLTFParser { /* GLTFREGISTRY */ -class GLTFRegistry { +export class GLTFRegistry { objects = {} get(key) { diff --git a/packages/engine/src/gltf/GLTFComponent.tsx b/packages/engine/src/gltf/GLTFComponent.tsx index 75600d17a1..27a61b0a3c 100644 --- a/packages/engine/src/gltf/GLTFComponent.tsx +++ b/packages/engine/src/gltf/GLTFComponent.tsx @@ -38,7 +38,7 @@ import { useEntityContext, useQuery } from '@etherealengine/ecs' -import { dispatchAction, getState, useHookstate } from '@etherealengine/hyperflux' +import { dispatchAction, getState, useHookstate, useMutableState } from '@etherealengine/hyperflux' import { FileLoader } from '../assets/loaders/base/FileLoader' import { BINARY_EXTENSION_HEADER_MAGIC, EXTENSIONS, GLTFBinaryExtension } from '../assets/loaders/gltf/GLTFExtensions' @@ -78,6 +78,7 @@ export const GLTFComponent = defineComponent({ const ResourceReactor = (props: { documentID: string; entity: Entity }) => { const resourceQuery = useQuery([SourceComponent, ResourcePendingComponent]) + const gltfDocumentState = useMutableState(GLTFDocumentState) const sourceEntities = useHookstate(SourceComponent.entitiesBySourceState[props.documentID]) useEffect(() => { @@ -109,7 +110,7 @@ const ResourceReactor = (props: { documentID: string; entity: Entity }) => { const percentage = total === 0 ? 100 : (progress / total) * 100 getMutableComponent(props.entity, GLTFComponent).progress.set(percentage) - }, [resourceQuery, sourceEntities]) + }, [resourceQuery, sourceEntities, gltfDocumentState[props.documentID]]) return null } diff --git a/packages/engine/src/gltf/GLTFDocumentState.ts b/packages/engine/src/gltf/GLTFDocumentState.ts index afef86140f..81afecd8a1 100644 --- a/packages/engine/src/gltf/GLTFDocumentState.ts +++ b/packages/engine/src/gltf/GLTFDocumentState.ts @@ -100,18 +100,18 @@ export const GLTFNodeState = defineState({ } }, - convertGltfToNodeDictionary: (gltf: GLTF.IGLTF) => { + convertGltfToNodeDictionary: (gltf: GLTF.IGLTF, source: string) => { const nodes: Record = {} const addNode = (nodeIndex: number, childIndex: number, parentUUID: EntityUUID | null) => { const node = gltf.nodes![nodeIndex] - const uuid = node.extensions?.[UUIDComponent.jsonID] as any as EntityUUID + const uuid = (node.extensions?.[UUIDComponent.jsonID] as any as EntityUUID) ?? source + '-' + nodeIndex if (uuid) { nodes[uuid] = { nodeIndex, childIndex, parentUUID } } else { /** @todo generate a globally deterministic UUID here */ - console.warn('Node does not have a UUID:', node) - return + // console.warn('Node does not have a UUID:', node) + // return } if (node.children) { for (let i = 0; i < node.children.length; i++) { @@ -129,7 +129,7 @@ export const GLTFNodeState = defineState({ for (let i = 0; i < gltf.scenes![0].nodes!.length; i++) { const nodeIndex = gltf.scenes![0].nodes![i] const node = gltf.nodes![nodeIndex] - const uuid = node.extensions?.[UUIDComponent.jsonID] as any as EntityUUID + const uuid = (node.extensions?.[UUIDComponent.jsonID] as any as EntityUUID) ?? source + '-' + nodeIndex if (uuid) { nodes[uuid] = { nodeIndex, diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts new file mode 100644 index 0000000000..5bf0ec7f1e --- /dev/null +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -0,0 +1,265 @@ +import { GLTF } from '@gltf-transform/core' +import { + Box3, + BufferAttribute, + BufferGeometry, + InterleavedBuffer, + InterleavedBufferAttribute, + LoaderUtils, + Sphere, + Vector3 +} from 'three' +import { FileLoader } from '../assets/loaders/base/FileLoader' +import { WEBGL_COMPONENT_TYPES, WEBGL_TYPE_SIZES } from '../assets/loaders/gltf/GLTFConstants' +import { getNormalizedComponentScale } from '../assets/loaders/gltf/GLTFLoaderFunctions' +import { GLTFParserOptions, GLTFRegistry } from '../assets/loaders/gltf/GLTFParser' + +// todo make this a state +const cache = new GLTFRegistry() + +const loadAccessor = (options: GLTFParserOptions, json: GLTF.IGLTF, accessorIndex: number) => { + const accessorDef = json.accessors![accessorIndex] + + if (accessorDef.bufferView === undefined && accessorDef.sparse === undefined) { + const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] + const TypedArray = WEBGL_COMPONENT_TYPES[accessorDef.componentType] + const normalized = accessorDef.normalized === true + + const array = new TypedArray(accessorDef.count * itemSize) + return Promise.resolve(new BufferAttribute(array, itemSize, normalized)) + } + + const pendingBufferViews = [] as Array | null> // todo + + if (accessorDef.bufferView !== undefined) { + pendingBufferViews.push(GLTFLoaderFunctions.loadBufferView(options, json, accessorDef.bufferView)) + } else { + pendingBufferViews.push(null) + } + + if (accessorDef.sparse !== undefined) { + pendingBufferViews.push(GLTFLoaderFunctions.loadBufferView(options, json, accessorDef.sparse.indices.bufferView)) + pendingBufferViews.push(GLTFLoaderFunctions.loadBufferView(options, json, accessorDef.sparse.values.bufferView)) + } + + return Promise.all(pendingBufferViews).then(function (bufferViews) { + const bufferView = bufferViews[0] + + const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] + const TypedArray = WEBGL_COMPONENT_TYPES[accessorDef.componentType] + + // For VEC3: itemSize is 3, elementBytes is 4, itemBytes is 12. + const elementBytes = TypedArray.BYTES_PER_ELEMENT + const itemBytes = elementBytes * itemSize + const byteOffset = accessorDef.byteOffset || 0 + const byteStride = + accessorDef.bufferView !== undefined ? json.bufferViews![accessorDef.bufferView].byteStride : undefined + const normalized = accessorDef.normalized === true + let array, bufferAttribute + + // The buffer is not interleaved if the stride is the item size in bytes. + if (byteStride && byteStride !== itemBytes) { + // Each "slice" of the buffer, as defined by 'count' elements of 'byteStride' bytes, gets its own InterleavedBuffer + // This makes sure that IBA.count reflects accessor.count properly + const ibSlice = Math.floor(byteOffset / byteStride) + const ibCacheKey = + 'InterleavedBuffer:' + + accessorDef.bufferView + + ':' + + accessorDef.componentType + + ':' + + ibSlice + + ':' + + accessorDef.count + let ib = cache.get(ibCacheKey) + + if (!ib) { + array = new TypedArray(bufferView!, ibSlice * byteStride, (accessorDef.count * byteStride) / elementBytes) + + // Integer parameters to IB/IBA are in array elements, not bytes. + ib = new InterleavedBuffer(array, byteStride / elementBytes) + + cache.add(ibCacheKey, ib) + } + + bufferAttribute = new InterleavedBufferAttribute( + ib, + itemSize, + (byteOffset % byteStride) / elementBytes, + normalized + ) + } else { + if (bufferView === null) { + array = new TypedArray(accessorDef.count * itemSize) + } else { + array = new TypedArray(bufferView, byteOffset, accessorDef.count * itemSize) + } + + bufferAttribute = new BufferAttribute(array, itemSize, normalized) + } + + // https://github.com/KhronosGroup/glTF/blob/master/specification/2.0/README.md#sparse-accessors + if (accessorDef.sparse !== undefined) { + const itemSizeIndices = WEBGL_TYPE_SIZES.SCALAR + const TypedArrayIndices = WEBGL_COMPONENT_TYPES[accessorDef.sparse.indices.componentType] + + const byteOffsetIndices = accessorDef.sparse.indices.byteOffset || 0 + const byteOffsetValues = accessorDef.sparse.values.byteOffset || 0 + + const sparseIndices = new TypedArrayIndices( + bufferViews[1]!, + byteOffsetIndices, + accessorDef.sparse.count * itemSizeIndices + ) + const sparseValues = new TypedArray(bufferViews[2]!, byteOffsetValues, accessorDef.sparse.count * itemSize) + + if (bufferView !== null) { + // Avoid modifying the original ArrayBuffer, if the bufferView wasn't initialized with zeroes. + bufferAttribute = new BufferAttribute( + bufferAttribute.array.slice(), + bufferAttribute.itemSize, + bufferAttribute.normalized + ) + } + + for (let i = 0, il = sparseIndices.length; i < il; i++) { + const index = sparseIndices[i] + + bufferAttribute.setX(index, sparseValues[i * itemSize]) + if (itemSize >= 2) bufferAttribute.setY(index, sparseValues[i * itemSize + 1]) + if (itemSize >= 3) bufferAttribute.setZ(index, sparseValues[i * itemSize + 2]) + if (itemSize >= 4) bufferAttribute.setW(index, sparseValues[i * itemSize + 3]) + if (itemSize >= 5) throw new Error('THREE.GLTFLoader: Unsupported itemSize in sparse BufferAttribute.') + } + } + + return bufferAttribute + }) +} + +const loadBufferView = async (options: GLTFParserOptions, json: GLTF.IGLTF, bufferViewIndex: number) => { + const bufferViewDef = json.bufferViews![bufferViewIndex] + + const buffer = await GLTFLoaderFunctions.loadBuffer(options, json, bufferViewDef.buffer) + + const byteLength = bufferViewDef.byteLength || 0 + const byteOffset = bufferViewDef.byteOffset || 0 + + return buffer.slice(byteOffset, byteOffset + byteLength) +} + +const loadBuffer = async (options: GLTFParserOptions, json: GLTF.IGLTF, bufferIndex: number) => { + const bufferDef = json.buffers![bufferIndex] + + /** @todo */ + // if (bufferDef.type && bufferDef.type !== 'arraybuffer') { + // throw new Error('THREE.GLTFLoader: ' + bufferDef.type + ' buffer type is not supported.') + // } + + // If present, GLB container is required to be the first buffer. + // if (bufferDef.uri === undefined && bufferIndex === 0) { + // return Promise.resolve(this.extensions[EXTENSIONS.KHR_BINARY_GLTF].body) + // } + + /** @todo use global file loader */ + const fileLoader = new FileLoader(options.manager) + fileLoader.setResponseType('arraybuffer') + if (options.crossOrigin === 'use-credentials') { + fileLoader.setWithCredentials(true) + } + + return new Promise(function (resolve, reject) { + fileLoader.load(LoaderUtils.resolveURL(bufferDef.uri!, options.path), resolve as any, undefined, function () { + reject(new Error('THREE.GLTFLoader: Failed to load buffer "' + bufferDef.uri + '".')) + }) + }) +} + +export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primitiveDef: GLTF.IMeshPrimitive) { + const attributes = primitiveDef.attributes + + const box = new Box3() + + if (attributes.POSITION !== undefined) { + const accessor = json.accessors![attributes.POSITION] + + const min = accessor.min + const max = accessor.max + + // glTF requires 'min' and 'max', but VRM (which extends glTF) currently ignores that requirement. + + if (min !== undefined && max !== undefined) { + box.set(new Vector3(min[0], min[1], min[2]), new Vector3(max[0], max[1], max[2])) + + if (accessor.normalized) { + const boxScale = getNormalizedComponentScale(WEBGL_COMPONENT_TYPES[accessor.componentType]) + box.min.multiplyScalar(boxScale) + box.max.multiplyScalar(boxScale) + } + } else { + console.warn('THREE.GLTFLoader: Missing min/max properties for accessor POSITION.') + + return + } + } else { + return + } + + const targets = primitiveDef.targets + + if (targets !== undefined) { + const maxDisplacement = new Vector3() + const vector = new Vector3() + + for (let i = 0, il = targets.length; i < il; i++) { + const target = targets[i] + + if (target.POSITION !== undefined) { + const accessor = json.accessors![target.POSITION] + const min = accessor.min + const max = accessor.max + + // glTF requires 'min' and 'max', but VRM (which extends glTF) currently ignores that requirement. + + if (min !== undefined && max !== undefined) { + // we need to get max of absolute components because target weight is [-1,1] + vector.setX(Math.max(Math.abs(min[0]), Math.abs(max[0]))) + vector.setY(Math.max(Math.abs(min[1]), Math.abs(max[1]))) + vector.setZ(Math.max(Math.abs(min[2]), Math.abs(max[2]))) + + if (accessor.normalized) { + const boxScale = getNormalizedComponentScale(WEBGL_COMPONENT_TYPES[accessor.componentType]) + vector.multiplyScalar(boxScale) + } + + // Note: this assumes that the sum of all weights is at most 1. This isn't quite correct - it's more conservative + // to assume that each target can have a max weight of 1. However, for some use cases - notably, when morph targets + // are used to implement key-frame animations and as such only two are active at a time - this results in very large + // boxes. So for now we make a box that's sometimes a touch too small but is hopefully mostly of reasonable size. + maxDisplacement.max(vector) + } else { + console.warn('THREE.GLTFLoader: Missing min/max properties for accessor POSITION.') + } + } + } + + // As per comment above this box isn't conservative, but has a reasonable size for a very large number of morph targets. + box.expandByVector(maxDisplacement) + } + + geometry.boundingBox = box + + const sphere = new Sphere() + + box.getCenter(sphere.center) + sphere.radius = box.min.distanceTo(box.max) / 2 + + geometry.boundingSphere = sphere +} + +export const GLTFLoaderFunctions = { + loadAccessor, + loadBufferView, + loadBuffer, + computeBounds +} diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 33c28713eb..1a6849cbb3 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -25,7 +25,19 @@ Ethereal Engine. All Rights Reserved. import { GLTF } from '@gltf-transform/core' import React, { useEffect, useLayoutEffect } from 'react' -import { Group, MathUtils, Matrix4, Quaternion, Vector3 } from 'three' +import { + BufferGeometry, + ColorManagement, + Group, + LinearSRGBColorSpace, + LoaderUtils, + MathUtils, + Matrix4, + Mesh, + MeshBasicMaterial, + Quaternion, + Vector3 +} from 'three' import { staticResourcePath } from '@etherealengine/common/src/schema.type.module' import { @@ -63,15 +75,23 @@ import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/component import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' import { VisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' -import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { + EntityTreeComponent, + getAncestorWithComponent +} from '@etherealengine/spatial/src/transform/components/EntityTree' import { EngineState } from '@etherealengine/spatial/src/EngineState' import { Physics } from '@etherealengine/spatial/src/physics/classes/Physics' import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' +import { ATTRIBUTES } from '../assets/loaders/gltf/GLTFConstants' +import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' +import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' +import { AssetLoaderState } from '../assets/state/AssetLoaderState' import { SourceComponent } from '../scene/components/SourceComponent' import { proxifyParentChildRelationships } from '../scene/functions/loadGLTFModel' import { GLTFComponent } from './GLTFComponent' import { GLTFDocumentState, GLTFModifiedState, GLTFNodeState, GLTFSnapshotAction } from './GLTFDocumentState' +import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' export const GLTFAssetState = defineState({ name: 'ee.engine.gltf.GLTFAssetState', @@ -336,7 +356,7 @@ const ChildGLTFReactor = (props: { source: string }) => { getMutableState(GLTFDocumentState)[source].set(data) // update the nodes dictionary - const nodesDictionary = GLTFNodeState.convertGltfToNodeDictionary(data) + const nodesDictionary = GLTFNodeState.convertGltfToNodeDictionary(data, source) getMutableState(GLTFNodeState)[source].set(nodesDictionary) }, [index]) @@ -385,8 +405,13 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: const parentEntity = UUIDComponent.useEntityByUUID(props.parentUUID) - const entity = useHookstate(() => { - const uuid = node.extensions.value?.[UUIDComponent.jsonID] as EntityUUID + const entityState = useHookstate(UndefinedEntity) + const entity = entityState.value + + useEffect(() => { + const uuid = + (node.extensions.value?.[UUIDComponent.jsonID] as EntityUUID) ?? + ((props.documentID + '-' + props.nodeIndex) as EntityUUID) const entity = UUIDComponent.getOrCreateEntityByUUID(uuid) setComponent(entity, UUIDComponent, uuid) @@ -406,6 +431,9 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: setComponent(entity, TransformComponent, { position, rotation, scale }) } + /** Always set visible extension if this is not an ECS node */ + if (!node.extensions.value?.[UUIDComponent.jsonID]) setComponent(entity, VisibleComponent) + // add all extensions for synchronous mount if (node.extensions.value) { for (const extension in node.extensions.value) { @@ -423,10 +451,8 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: setComponent(entity, Object3DComponent, obj3d) } - return entity - }).value + entityState.set(entity) - useEffect(() => { return () => { //check if entity is in some other document if (hasComponent(entity, UUIDComponent)) { @@ -474,9 +500,9 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: return ( <> - {/* {node.mesh.value && ( + {typeof node.mesh.get(NO_PROXY) === 'number' && ( - )} */} + )} {node.extensions.value && Object.keys(node.extensions.get(NO_PROXY)!).map((extension) => ( { const documentState = useMutableState(GLTFDocumentState)[props.documentID] - const nodes = documentState.nodes! // as State + const nodes = documentState.nodes!.get(NO_PROXY)! const node = nodes[props.nodeIndex]! const extension = node.extensions![props.extension] @@ -520,49 +546,195 @@ const ExtensionReactor = (props: { entity: Entity; extension: string; nodeIndex: useLayoutEffect(() => { const Component = ComponentJSONIDMap.get(props.extension) if (!Component) return console.warn('no component found for extension', props.extension) - setComponent(props.entity, Component, extension.get(NO_PROXY_STEALTH)) + setComponent(props.entity, Component, extension) }, [extension]) return null } -// const MeshReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { -// const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) -// const nodes = documentState.nodes! as State -// const node = nodes[props.nodeIndex]! - -// const mesh = documentState.meshes![node.mesh.value!] as State - -// return ( -// <> -// {mesh.primitives.value.map((primitive, index) => ( -// -// ))} -// -// ) -// } - -// const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; documentID: string; entity: Entity }) => { -// const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) -// const nodes = documentState.nodes! as State -// const node = nodes[props.nodeIndex]! - -// const primitive = documentState.meshes![node.mesh.value!].primitives[props.primitiveIndex] - -// useEffect(() => { -// /** TODO implement all mesh types */ -// }, [primitive]) - -// return null -// } +const MeshReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + useEffect(() => { + setComponent(props.entity, VisibleComponent) + }, []) + + return ( + <> + {mesh.primitives.map((primitive, index) => ( + + ))} + + ) +} + +const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; documentID: string; entity: Entity }) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const primitive = mesh.primitives[props.primitiveIndex] + + const geometry = useHookstate(null as null | BufferGeometry) + + useEffect(() => { + if (primitive.extensions && primitive.extensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION]) { + /** @todo */ + // extensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION].decodePrimitive(primitive, parser).then((geom) => { + // geometry.set(geom) + // }) + } else { + geometry.set(new BufferGeometry()) + } + + if (ColorManagement.workingColorSpace !== LinearSRGBColorSpace && 'COLOR_0' in primitive.attributes) { + console.warn( + `THREE.GLTFLoader: Converting vertex colors from "srgb-linear" to "${ColorManagement.workingColorSpace}" not supported.` + ) + } + }, [primitive.extensions]) + + useEffect(() => { + const mesh = new Mesh(geometry.value as BufferGeometry, new MeshBasicMaterial()) + setComponent(props.entity, MeshComponent, mesh) + addObjectToGroup(props.entity, mesh) + + GLTFLoaderFunctions.computeBounds( + documentState.get(NO_PROXY) as GLTF.IGLTF, + geometry.value as BufferGeometry, + primitive as GLTF.IMeshPrimitive + ) + + return () => { + removeComponent(props.entity, MeshComponent) + } + }, [geometry]) + + if (!geometry.value) return null + + return ( + <> + {Object.keys(primitive.attributes).map((attribute, index) => ( + + ))} + {typeof primitive.indices === 'number' && ( + + )} + + ) +} + +const PrimitiveAttributeReactor = (props: { + geometry: BufferGeometry + attribute: string + primitiveIndex: number + nodeIndex: number + documentID: string + entity: Entity +}) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const primitive = mesh.primitives[props.primitiveIndex] + + const attribute = primitive.attributes[props.attribute] + + useEffect(() => { + const threeAttributeName = ATTRIBUTES[props.attribute] || props.attribute.toLowerCase() + + // Skip attributes already provided by e.g. Draco extension. + if (threeAttributeName in props.geometry.attributes) return + + // accessor + GLTFLoaderFunctions.loadAccessor( + getParserOptions(props.entity), + documentState.get(NO_PROXY) as GLTF.IGLTF, + attribute + ).then((accessor) => { + props.geometry.setAttribute(threeAttributeName, accessor) + }) + }, [attribute, props.geometry]) + + return null +} + +const PrimitiveIndicesAttributeReactor = (props: { + geometry: BufferGeometry + primitiveIndex: number + nodeIndex: number + documentID: string + entity: Entity +}) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const primitive = mesh.primitives[props.primitiveIndex] + + useEffect(() => { + // accessor + GLTFLoaderFunctions.loadAccessor( + getParserOptions(props.entity), + documentState.get(NO_PROXY) as GLTF.IGLTF, + primitive.indices! + ).then((accessor) => { + props.geometry.setIndex(accessor) + }) + }, [primitive.indices, props.geometry]) + + return null +} /** * TODO figure out how to support extensions that change the behaviour of these reactors * - we pretty much have to add a new API for each dependency type, like how the GLTFLoader does */ + +const getParserOptions = (entity: Entity) => { + const gltfEntity = getAncestorWithComponent(entity, GLTFComponent) + const url = getComponent(gltfEntity, GLTFComponent).src + const gltfLoader = getState(AssetLoaderState).gltfLoader + return { + url, + path: LoaderUtils.extractUrlBase(url), + crossOrigin: gltfLoader.crossOrigin, + requestHeader: gltfLoader.requestHeader, + manager: gltfLoader.manager, + ktx2Loader: gltfLoader.ktx2Loader, + meshoptDecoder: gltfLoader.meshoptDecoder + } as GLTFParserOptions +} From e6d4e0ead25ded1c122060154d12904dc0c966a4 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sat, 27 Jul 2024 12:10:19 +1000 Subject: [PATCH 02/47] license --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 5bf0ec7f1e..05e2310cfa 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -1,3 +1,28 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + import { GLTF } from '@gltf-transform/core' import { Box3, From a09f2d52000454f4e8821b059cf22ca7befacb4c Mon Sep 17 00:00:00 2001 From: HexaField Date: Sat, 27 Jul 2024 22:06:55 +1000 Subject: [PATCH 03/47] can load binary buffers, draco and quantized primitives --- .../src/assets/loaders/gltf/GLTFParser.ts | 1 + packages/engine/src/gltf/GLTFComponent.tsx | 69 +++++++++-- packages/engine/src/gltf/GLTFExtensions.ts | 63 ++++++++++ .../engine/src/gltf/GLTFLoaderFunctions.ts | 17 +-- packages/engine/src/gltf/GLTFState.tsx | 114 ++++++++++++++---- 5 files changed, 224 insertions(+), 40 deletions(-) create mode 100644 packages/engine/src/gltf/GLTFExtensions.ts diff --git a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts index 074eb1be57..31cbc139b5 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts @@ -124,6 +124,7 @@ declare module '@gltf-transform/core/dist/types/gltf.d.ts' { } export type GLTFParserOptions = { + body: null | ArrayBuffer crossOrigin: 'anonymous' | string ktx2Loader: KTX2Loader manager: LoadingManager | any diff --git a/packages/engine/src/gltf/GLTFComponent.tsx b/packages/engine/src/gltf/GLTFComponent.tsx index 27a61b0a3c..b8b72c8c67 100644 --- a/packages/engine/src/gltf/GLTFComponent.tsx +++ b/packages/engine/src/gltf/GLTFComponent.tsx @@ -41,7 +41,11 @@ import { import { dispatchAction, getState, useHookstate, useMutableState } from '@etherealengine/hyperflux' import { FileLoader } from '../assets/loaders/base/FileLoader' -import { BINARY_EXTENSION_HEADER_MAGIC, EXTENSIONS, GLTFBinaryExtension } from '../assets/loaders/gltf/GLTFExtensions' +import { + BINARY_EXTENSION_CHUNK_TYPES, + BINARY_EXTENSION_HEADER_LENGTH, + BINARY_EXTENSION_HEADER_MAGIC +} from '../assets/loaders/gltf/GLTFExtensions' import { SourceComponent } from '../scene/components/SourceComponent' import { SceneJsonType } from '../scene/types/SceneTypes' import { migrateSceneJSONToGLTF } from './convertJsonToGLTF' @@ -55,7 +59,7 @@ export const GLTFComponent = defineComponent({ return { src: '', // internals - extensions: {}, + body: null as null | ArrayBuffer, progress: 0 } }, @@ -153,14 +157,13 @@ const useGLTFDocument = (url: string, entity: Entity) => { if (magic === BINARY_EXTENSION_HEADER_MAGIC) { try { - /** TODO we will need to refactor and persist this */ - state.extensions.merge({ [EXTENSIONS.KHR_BINARY_GLTF]: new GLTFBinaryExtension(data) }) + const { json: jsonContent, body } = parseBinaryData(data) + state.body.set(body) + json = jsonContent } catch (error) { if (onError) onError(error) return } - - json = JSON.parse(state.extensions.value[EXTENSIONS.KHR_BINARY_GLTF].content) } else { json = JSON.parse(textDecoder.decode(data)) } @@ -192,9 +195,57 @@ const useGLTFDocument = (url: string, entity: Entity) => { return () => { abortController.abort() if (!hasComponent(entity, GLTFComponent)) return - state.merge({ - extensions: {} - }) + state.body.set(null) } }, [url]) } + +export const parseBinaryData = (data) => { + const headerView = new DataView(data, 0, BINARY_EXTENSION_HEADER_LENGTH) + const textDecoder = new TextDecoder() + + const header = { + magic: textDecoder.decode(new Uint8Array(data.slice(0, 4))), + version: headerView.getUint32(4, true), + length: headerView.getUint32(8, true) + } + + if (header.magic !== BINARY_EXTENSION_HEADER_MAGIC) { + throw new Error('THREE.GLTFLoader: Unsupported glTF-Binary header.') + } else if (header.version < 2.0) { + throw new Error('THREE.GLTFLoader: Legacy binary file detected.') + } + + const chunkContentsLength = header.length - BINARY_EXTENSION_HEADER_LENGTH + const chunkView = new DataView(data, BINARY_EXTENSION_HEADER_LENGTH) + let chunkIndex = 0 + + let content = null as string | null + let body = null as ArrayBuffer | null + + while (chunkIndex < chunkContentsLength) { + const chunkLength = chunkView.getUint32(chunkIndex, true) + chunkIndex += 4 + + const chunkType = chunkView.getUint32(chunkIndex, true) + chunkIndex += 4 + + if (chunkType === BINARY_EXTENSION_CHUNK_TYPES.JSON) { + const contentArray = new Uint8Array(data, BINARY_EXTENSION_HEADER_LENGTH + chunkIndex, chunkLength) + content = textDecoder.decode(contentArray) + } else if (chunkType === BINARY_EXTENSION_CHUNK_TYPES.BIN) { + const byteOffset = BINARY_EXTENSION_HEADER_LENGTH + chunkIndex + body = data.slice(byteOffset, byteOffset + chunkLength) + } + + // Clients must ignore chunks with unknown types. + + chunkIndex += chunkLength + } + + if (content === null) { + throw new Error('THREE.GLTFLoader: JSON content not found.') + } + + return { json: JSON.parse(content), body } +} diff --git a/packages/engine/src/gltf/GLTFExtensions.ts b/packages/engine/src/gltf/GLTFExtensions.ts new file mode 100644 index 0000000000..5d088f0bcc --- /dev/null +++ b/packages/engine/src/gltf/GLTFExtensions.ts @@ -0,0 +1,63 @@ +import { getState } from '@etherealengine/hyperflux' +import { GLTF } from '@gltf-transform/core' +import { BufferGeometry, NormalBufferAttributes } from 'three' +import { ATTRIBUTES, WEBGL_COMPONENT_TYPES } from '../assets/loaders/gltf/GLTFConstants' +import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' +import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' +import { AssetLoaderState } from '../assets/state/AssetLoaderState' +import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' + +const khr_draco_mesh_compression = { + decodePrimitive(options: GLTFParserOptions, json: GLTF.IGLTF, primitive: GLTF.IMeshPrimitive) { + const dracoLoader = getState(AssetLoaderState).gltfLoader.dracoLoader! + const dracoMeshCompressionExtension = primitive.extensions![EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] as any + const bufferViewIndex = dracoMeshCompressionExtension.bufferView + const gltfAttributeMap = dracoMeshCompressionExtension.attributes + const threeAttributeMap = {} as { [key: string]: string } + const attributeNormalizedMap = {} as { [key: string]: boolean } + const attributeTypeMap = {} as { [key: string]: string } + + for (const attributeName in gltfAttributeMap) { + const threeAttributeName = ATTRIBUTES[attributeName] || attributeName.toLowerCase() + + threeAttributeMap[threeAttributeName] = gltfAttributeMap[attributeName] + } + + for (const attributeName in primitive.attributes) { + const threeAttributeName = ATTRIBUTES[attributeName] || attributeName.toLowerCase() + + if (gltfAttributeMap[attributeName] !== undefined) { + // @ts-ignore -- TODO type extensions + const accessorDef = json.accessors[primitive.attributes[attributeName]] + const componentType = WEBGL_COMPONENT_TYPES[accessorDef.componentType] + + attributeTypeMap[threeAttributeName] = componentType.name + attributeNormalizedMap[threeAttributeName] = accessorDef.normalized === true + } + } + + return GLTFLoaderFunctions.loadBufferView(options, json, bufferViewIndex).then(function (bufferView) { + return new Promise>(function (resolve) { + dracoLoader.preload().decodeDracoFile( + bufferView, + function (geometry) { + for (const attributeName in geometry.attributes) { + const attribute = geometry.attributes[attributeName] + const normalized = attributeNormalizedMap[attributeName] + + if (normalized !== undefined) attribute.normalized = normalized + } + + resolve(geometry) + }, + threeAttributeMap, + attributeTypeMap + ) + }) + }) + } +} + +export const GLTFExtensions = { + [EXTENSIONS.KHR_DRACO_MESH_COMPRESSION]: khr_draco_mesh_compression +} diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 05e2310cfa..8309106bab 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -176,17 +176,16 @@ const loadBufferView = async (options: GLTFParserOptions, json: GLTF.IGLTF, buff const loadBuffer = async (options: GLTFParserOptions, json: GLTF.IGLTF, bufferIndex: number) => { const bufferDef = json.buffers![bufferIndex] - /** @todo */ - // if (bufferDef.type && bufferDef.type !== 'arraybuffer') { - // throw new Error('THREE.GLTFLoader: ' + bufferDef.type + ' buffer type is not supported.') - // } + if (bufferDef.type && bufferDef.type !== 'arraybuffer') { + throw new Error('THREE.GLTFLoader: ' + bufferDef.type + ' buffer type is not supported.') + } // If present, GLB container is required to be the first buffer. - // if (bufferDef.uri === undefined && bufferIndex === 0) { - // return Promise.resolve(this.extensions[EXTENSIONS.KHR_BINARY_GLTF].body) - // } + if (bufferDef.uri === undefined && bufferIndex === 0) { + return Promise.resolve(options.body!) + } - /** @todo use global file loader */ + /** @todo use a global file loader */ const fileLoader = new FileLoader(options.manager) fileLoader.setResponseType('arraybuffer') if (options.crossOrigin === 'use-credentials') { @@ -203,6 +202,8 @@ const loadBuffer = async (options: GLTFParserOptions, json: GLTF.IGLTF, bufferIn export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primitiveDef: GLTF.IMeshPrimitive) { const attributes = primitiveDef.attributes + console.log('computeBounds', json, geometry, primitiveDef) + const box = new Box3() if (attributes.POSITION !== undefined) { diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 1a6849cbb3..25efdd2ad6 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -85,12 +85,14 @@ import { Physics } from '@etherealengine/spatial/src/physics/classes/Physics' import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' import { ATTRIBUTES } from '../assets/loaders/gltf/GLTFConstants' import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' +import { assignExtrasToUserData } from '../assets/loaders/gltf/GLTFLoaderFunctions' import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' import { AssetLoaderState } from '../assets/state/AssetLoaderState' import { SourceComponent } from '../scene/components/SourceComponent' import { proxifyParentChildRelationships } from '../scene/functions/loadGLTFModel' import { GLTFComponent } from './GLTFComponent' import { GLTFDocumentState, GLTFModifiedState, GLTFNodeState, GLTFSnapshotAction } from './GLTFDocumentState' +import { GLTFExtensions } from './GLTFExtensions' import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' export const GLTFAssetState = defineState({ @@ -340,12 +342,8 @@ const ChildGLTFReactor = (props: { source: string }) => { const index = useHookstate(getMutableState(GLTFSnapshotState)[source].index).value - useLayoutEffect(() => { - return () => { - getMutableState(GLTFDocumentState)[source].set(none) - getMutableState(GLTFNodeState)[source].set(none) - } - }, []) + const entity = useHookstate(getMutableState(GLTFSourceState)[source]).value + const parentUUID = useComponent(entity, UUIDComponent).value useLayoutEffect(() => { // update the modified state @@ -360,8 +358,12 @@ const ChildGLTFReactor = (props: { source: string }) => { getMutableState(GLTFNodeState)[source].set(nodesDictionary) }, [index]) - const entity = useHookstate(getMutableState(GLTFSourceState)[source]).value - const parentUUID = useComponent(entity, UUIDComponent).value + useLayoutEffect(() => { + return () => { + getMutableState(GLTFDocumentState)[source].set(none) + getMutableState(GLTFNodeState)[source].set(none) + } + }, []) return } @@ -496,6 +498,30 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: setComponent(entity, TransformComponent, { position, rotation, scale }) }, [entity, node.matrix]) + useLayoutEffect(() => { + if (!entity) return + + if (!node.translation.value) return + const position = new Vector3().fromArray(node.translation.value) + setComponent(entity, TransformComponent, { position }) + }, [entity, node.translation]) + + useLayoutEffect(() => { + if (!entity) return + + if (!node.rotation.value) return + const rotation = new Quaternion().fromArray(node.rotation.value) + setComponent(entity, TransformComponent, { rotation }) + }, [entity, node.rotation]) + + useLayoutEffect(() => { + if (!entity) return + + if (!node.scale.value) return + const scale = new Vector3().fromArray(node.scale.value) + setComponent(entity, TransformComponent, { scale }) + }, [entity, node.scale]) + if (!entity) return null return ( @@ -590,12 +616,17 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do const geometry = useHookstate(null as null | BufferGeometry) + const hasDracoCompression = primitive.extensions && primitive.extensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] + useEffect(() => { - if (primitive.extensions && primitive.extensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION]) { + if (hasDracoCompression) { /** @todo */ - // extensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION].decodePrimitive(primitive, parser).then((geom) => { - // geometry.set(geom) - // }) + const options = getParserOptions(props.entity) + GLTFExtensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] + .decodePrimitive(options, documentState.get(NO_PROXY) as GLTF.IGLTF, primitive as GLTF.IMeshPrimitive) + .then((geom) => { + geometry.set(geom) + }) } else { geometry.set(new BufferGeometry()) } @@ -608,10 +639,14 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do }, [primitive.extensions]) useEffect(() => { + if (!geometry.value) return + const mesh = new Mesh(geometry.value as BufferGeometry, new MeshBasicMaterial()) setComponent(props.entity, MeshComponent, mesh) addObjectToGroup(props.entity, mesh) + assignExtrasToUserData(geometry, primitive as GLTF.IMeshPrimitive) + GLTFLoaderFunctions.computeBounds( documentState.get(NO_PROXY) as GLTF.IGLTF, geometry.value as BufferGeometry, @@ -627,18 +662,27 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do return ( <> - {Object.keys(primitive.attributes).map((attribute, index) => ( - - ))} - {typeof primitive.indices === 'number' && ( + )} + {!hasDracoCompression && + Object.keys(primitive.attributes).map((attribute, index) => ( + + ))} + {!hasDracoCompression && typeof primitive.indices === 'number' && ( { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const primitive = mesh.primitives[props.primitiveIndex] + + const material = primitive.material + + useEffect(() => { + // const material = GLTFLoaderFunctions.createMaterial( + // getParserOptions(props.entity), + // documentState.get(NO_PROXY) as GLTF.IGLTF, + // material + // ) + }, []) + + return null +} + /** * TODO figure out how to support extensions that change the behaviour of these reactors * - we pretty much have to add a new API for each dependency type, like how the GLTFLoader does @@ -726,11 +793,12 @@ const PrimitiveIndicesAttributeReactor = (props: { const getParserOptions = (entity: Entity) => { const gltfEntity = getAncestorWithComponent(entity, GLTFComponent) - const url = getComponent(gltfEntity, GLTFComponent).src + const gltfComponent = getComponent(gltfEntity, GLTFComponent) const gltfLoader = getState(AssetLoaderState).gltfLoader return { - url, - path: LoaderUtils.extractUrlBase(url), + url: gltfComponent.src, + path: LoaderUtils.extractUrlBase(gltfComponent.src), + body: gltfComponent.body, crossOrigin: gltfLoader.crossOrigin, requestHeader: gltfLoader.requestHeader, manager: gltfLoader.manager, From ac91fe3ef4427f2d4346d891d23a19ef74da30a7 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sat, 27 Jul 2024 22:07:04 +1000 Subject: [PATCH 04/47] license --- packages/engine/src/gltf/GLTFExtensions.ts | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/engine/src/gltf/GLTFExtensions.ts b/packages/engine/src/gltf/GLTFExtensions.ts index 5d088f0bcc..a3b79300d2 100644 --- a/packages/engine/src/gltf/GLTFExtensions.ts +++ b/packages/engine/src/gltf/GLTFExtensions.ts @@ -1,3 +1,28 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + import { getState } from '@etherealengine/hyperflux' import { GLTF } from '@gltf-transform/core' import { BufferGeometry, NormalBufferAttributes } from 'three' From 7ee811162e3afcd47a56fbf7bcdc9ae6e282beb5 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 28 Jul 2024 14:31:38 +1000 Subject: [PATCH 05/47] basic material support --- .../src/assets/loaders/gltf/GLTFParser.ts | 2 +- .../engine/src/gltf/GLTFLoaderFunctions.ts | 419 +++++++++++++++++- packages/engine/src/gltf/GLTFState.tsx | 27 +- 3 files changed, 432 insertions(+), 16 deletions(-) diff --git a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts index 31cbc139b5..60651b0c97 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts @@ -98,7 +98,7 @@ import { } from './GLTFLoaderFunctions' import { KTX2Loader } from './KTX2Loader' -function getImageURIMimeType(uri) { +export function getImageURIMimeType(uri) { if (uri.search(/\.jpe?g($|\?)/i) > 0 || uri.search(/^data\:image\/jpeg/) === 0) return 'image/jpeg' if (uri.search(/\.webp($|\?)/i) > 0 || uri.search(/^data\:image\/webp/) === 0) return 'image/webp' diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 8309106bab..179db79dae 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -28,16 +28,41 @@ import { Box3, BufferAttribute, BufferGeometry, + Color, + DoubleSide, + ImageBitmapLoader, InterleavedBuffer, InterleavedBufferAttribute, + LinearFilter, + LinearMipmapLinearFilter, + LinearSRGBColorSpace, LoaderUtils, + MeshBasicMaterial, + MeshStandardMaterial, + RepeatWrapping, + SRGBColorSpace, Sphere, + Texture, + TextureLoader, + Vector2, Vector3 } from 'three' import { FileLoader } from '../assets/loaders/base/FileLoader' -import { WEBGL_COMPONENT_TYPES, WEBGL_TYPE_SIZES } from '../assets/loaders/gltf/GLTFConstants' -import { getNormalizedComponentScale } from '../assets/loaders/gltf/GLTFLoaderFunctions' -import { GLTFParserOptions, GLTFRegistry } from '../assets/loaders/gltf/GLTFParser' +import { + ALPHA_MODES, + WEBGL_COMPONENT_TYPES, + WEBGL_FILTERS, + WEBGL_TYPE_SIZES, + WEBGL_WRAPPINGS +} from '../assets/loaders/gltf/GLTFConstants' +import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' +import { + addUnknownExtensionsToUserData, + assignExtrasToUserData, + getNormalizedComponentScale +} from '../assets/loaders/gltf/GLTFLoaderFunctions' +import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' +import { GLTFExtensions } from './GLTFExtensions' // todo make this a state const cache = new GLTFRegistry() @@ -202,8 +227,6 @@ const loadBuffer = async (options: GLTFParserOptions, json: GLTF.IGLTF, bufferIn export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primitiveDef: GLTF.IMeshPrimitive) { const attributes = primitiveDef.attributes - console.log('computeBounds', json, geometry, primitiveDef) - const box = new Box3() if (attributes.POSITION !== undefined) { @@ -283,9 +306,393 @@ export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primit geometry.boundingSphere = sphere } +/** + * Specification: https://github.com/KhronosGroup/glTF/blob/master/specification/2.0/README.md#materials + * @param {number} materialIndex + * @return {Promise} + */ +const loadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialIndex: number) => { + const materialDef = json.materials![materialIndex] + + let materialType + const materialParams = {} as any // todo + const materialExtensions = materialDef.extensions || {} + + const pending = [] as any[] // todo + + if (!materialExtensions[EXTENSIONS.EE_MATERIAL] && materialExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT]) { + /** @todo */ + // const kmuExtension = extensions[EXTENSIONS.KHR_MATERIALS_UNLIT] + // materialType = kmuExtension.getMaterialType() + // pending.push(kmuExtension.extendParams(materialParams, materialDef, parser)) + } else { + // Specification: + // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#metallic-roughness-material + + const metallicRoughness = materialDef.pbrMetallicRoughness || {} + + materialParams.color = new Color(1.0, 1.0, 1.0) + materialParams.opacity = 1.0 + + if (Array.isArray(metallicRoughness.baseColorFactor)) { + const array = metallicRoughness.baseColorFactor + + materialParams.color.setRGB(array[0], array[1], array[2], LinearSRGBColorSpace) + materialParams.opacity = array[3] + } + + if (metallicRoughness.baseColorTexture !== undefined) { + pending.push( + GLTFLoaderFunctions.assignTexture( + options, + json, + materialParams, + 'map', + metallicRoughness.baseColorTexture, + SRGBColorSpace + ) + ) + } + + materialParams.metalness = metallicRoughness.metallicFactor !== undefined ? metallicRoughness.metallicFactor : 1.0 + materialParams.roughness = metallicRoughness.roughnessFactor !== undefined ? metallicRoughness.roughnessFactor : 1.0 + + if (metallicRoughness.metallicRoughnessTexture !== undefined) { + pending.push( + GLTFLoaderFunctions.assignTexture( + options, + json, + materialParams, + 'metalnessMap', + metallicRoughness.metallicRoughnessTexture + ) + ) + pending.push( + GLTFLoaderFunctions.assignTexture( + options, + json, + materialParams, + 'roughnessMap', + metallicRoughness.metallicRoughnessTexture + ) + ) + } + + /** @todo expose 'getMaterialType' API */ + materialType = MeshStandardMaterial + + /** @todo expose API */ + // pending.push( + // Promise.all( + // this._invokeAll(function (ext) { + // return ext.extendMaterialParams && ext.extendMaterialParams(materialIndex, materialParams) + // }) + // ) + // ) + } + + if (materialDef.doubleSided === true) { + materialParams.side = DoubleSide + } + + const alphaMode = materialDef.alphaMode || ALPHA_MODES.OPAQUE + + if (alphaMode === ALPHA_MODES.BLEND) { + materialParams.transparent = true + + // See: https://github.com/mrdoob/three.js/issues/17706 + if (materialParams.depthWrite === undefined) { + materialParams.depthWrite = false + } + } else { + materialParams.transparent = false + + if (alphaMode === ALPHA_MODES.MASK) { + materialParams.alphaTest = materialDef.alphaCutoff !== undefined ? materialDef.alphaCutoff : 0.5 + } + } + + if (materialDef.normalTexture !== undefined && materialType !== MeshBasicMaterial) { + pending.push( + GLTFLoaderFunctions.assignTexture(options, json, materialParams, 'normalMap', materialDef.normalTexture) + ) + + materialParams.normalScale = new Vector2(1, 1) + + if (materialDef.normalTexture.scale !== undefined) { + const scale = materialDef.normalTexture.scale + + materialParams.normalScale.set(scale, scale) + } + } + + if (materialDef.occlusionTexture !== undefined && materialType !== MeshBasicMaterial) { + pending.push( + GLTFLoaderFunctions.assignTexture(options, json, materialParams, 'aoMap', materialDef.occlusionTexture) + ) + + if (materialDef.occlusionTexture.strength !== undefined) { + materialParams.aoMapIntensity = materialDef.occlusionTexture.strength + } + } + + if (materialDef.emissiveFactor !== undefined && materialType !== MeshBasicMaterial) { + const emissiveFactor = materialDef.emissiveFactor + materialParams.emissive = new Color().setRGB( + emissiveFactor[0], + emissiveFactor[1], + emissiveFactor[2], + LinearSRGBColorSpace + ) + } + + if (materialDef.emissiveTexture !== undefined && materialType !== MeshBasicMaterial) { + pending.push( + GLTFLoaderFunctions.assignTexture( + options, + json, + materialParams, + 'emissiveMap', + materialDef.emissiveTexture, + SRGBColorSpace + ) + ) + } + + return Promise.all(pending).then(function () { + const material = new materialType(materialParams) + + if (materialDef.name) material.name = materialDef.name + + assignExtrasToUserData(material, materialDef) + + /** @todo */ + // parser.associations.set(material, { materials: materialIndex }) + + if (materialDef.extensions) addUnknownExtensionsToUserData(GLTFExtensions, material, materialDef) + + return material + }) +} + +/** + * Asynchronously assigns a texture to the given material parameters. + * @param {Object} materialParams + * @param {string} mapName + * @param {Object} mapDef + * @return {Promise} + */ +const assignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, materialParams, mapName, mapDef, colorSpace?) => { + // eslint-disable-next-line @typescript-eslint/no-this-alias + + return GLTFLoaderFunctions.loadTexture(options, json, mapDef.index).then(function (texture) { + if (!texture) return null + + if (mapDef.texCoord !== undefined && mapDef.texCoord > 0) { + texture = texture.clone() + texture.channel = mapDef.texCoord + } + + if (GLTFExtensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM]) { + const transform = + mapDef.extensions !== undefined ? mapDef.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM] : undefined + + if (transform) { + /** @todo */ + // const gltfReference = parser.associations.get(texture) + // texture = parser.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM].extendTexture(texture, transform) + // parser.associations.set(texture, gltfReference) + } + } + + if (colorSpace !== undefined) { + texture.colorSpace = colorSpace + } + + materialParams[mapName] = texture + + return texture + }) +} + +/** + * Specification: https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#textures + * @param {number} textureIndex + * @return {Promise} + */ +const loadTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, textureIndex: number) => { + const textureDef = json.textures![textureIndex] + const sourceIndex = textureDef.source! + const sourceDef = json.images![sourceIndex] + + let isSafari = false + let isFirefox = false + let firefoxVersion = -1 as any // ??? + + if (typeof navigator !== 'undefined') { + isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent) === true + isFirefox = navigator.userAgent.indexOf('Firefox') > -1 + firefoxVersion = isFirefox ? navigator.userAgent.match(/Firefox\/([0-9]+)\./)![1] : -1 + } + + let textureLoader + + /** @todo make global loader */ + if (typeof createImageBitmap === 'undefined' || isSafari || (isFirefox && firefoxVersion < 98)) { + textureLoader = new TextureLoader(options.manager) + } else { + textureLoader = new ImageBitmapLoader(options.manager) + } + + textureLoader.setCrossOrigin(options.crossOrigin) + textureLoader.setRequestHeader(options.requestHeader) + + let loader = textureLoader + + if (sourceDef.uri) { + const handler = options.manager.getHandler(sourceDef.uri) + if (handler !== null) loader = handler + } + + return GLTFLoaderFunctions.loadTextureImage(options, json, textureIndex, sourceIndex, loader) +} + +const textureCache = {} as any // todo + +const loadTextureImage = ( + options: GLTFParserOptions, + json: GLTF.IGLTF, + textureIndex: number, + sourceIndex: number, + loader +) => { + // eslint-disable-next-line @typescript-eslint/no-this-alias + + const textureDef = json.textures![textureIndex] + const sourceDef = json.images![sourceIndex] + + const cacheKey = (sourceDef.uri || sourceDef.bufferView) + ':' + textureDef.sampler + + if (textureCache[cacheKey]) { + // See https://github.com/mrdoob/three.js/issues/21559. + return textureCache[cacheKey] + } + + const promise = GLTFLoaderFunctions.loadImageSource(options, json, sourceIndex, loader) + .then(function (texture) { + texture.flipY = false + + texture.name = textureDef.name || sourceDef.name || '' + + if ( + texture.name === '' && + typeof sourceDef.uri === 'string' && + sourceDef.uri.startsWith('data:image/') === false + ) { + texture.name = sourceDef.uri + } + + const samplers = json.samplers || {} + const sampler = samplers[textureDef.sampler!] || {} + + texture.magFilter = WEBGL_FILTERS[sampler.magFilter] || LinearFilter + texture.minFilter = WEBGL_FILTERS[sampler.minFilter] || LinearMipmapLinearFilter + texture.wrapS = WEBGL_WRAPPINGS[sampler.wrapS] || RepeatWrapping + texture.wrapT = WEBGL_WRAPPINGS[sampler.wrapT] || RepeatWrapping + + /** @todo */ + // parser.associations.set(texture, { textures: textureIndex }) + + return texture + }) + .catch(function (error) { + console.error('THREE.GLTFLoader: Error in texture onLoad for texture', sourceDef) + console.error(error) + throw error + return null + }) + + textureCache[cacheKey] = promise + + return promise +} + +const sourceCache = {} as any // todo + +const loadImageSource = (options: GLTFParserOptions, json: GLTF.IGLTF, sourceIndex, loader) => { + // eslint-disable-next-line @typescript-eslint/no-this-alias + + if (sourceCache[sourceIndex] !== undefined) { + return sourceCache[sourceIndex].then((texture) => texture.clone()) + } + + const sourceDef = json.images![sourceIndex] + + const URL = self.URL || self.webkitURL + + let sourceURI = sourceDef.uri || ('' as string | Promise) + let isObjectURL = false + + if (sourceDef.bufferView !== undefined) { + // Load binary image data from bufferView, if provided. + + sourceURI = GLTFLoaderFunctions.loadBufferView(options, json, sourceDef.bufferView).then(function (bufferView) { + isObjectURL = true + const blob = new Blob([bufferView], { type: sourceDef.mimeType }) + sourceURI = URL.createObjectURL(blob) + return sourceURI + }) + } else if (sourceDef.uri === undefined) { + throw new Error('THREE.GLTFLoader: Image ' + sourceIndex + ' is missing URI and bufferView') + } + + const promise = Promise.resolve(sourceURI) + .then(function (sourceURI) { + return new Promise(function (resolve, reject) { + let onLoad = resolve + + if (loader.isImageBitmapLoader === true) { + onLoad = function (imageBitmap) { + const texture = new Texture(imageBitmap) + texture.needsUpdate = true + + resolve(texture) + } + } + + loader.load(LoaderUtils.resolveURL(sourceURI, options.path), onLoad, undefined, reject) + }) + }) + .then(function (texture) { + // Clean up resources and configure Texture. + + if (isObjectURL === true) { + URL.revokeObjectURL(sourceURI as string) + } else { + texture.userData.src = sourceURI + } + + texture.userData.mimeType = sourceDef.mimeType || getImageURIMimeType(sourceDef.uri) + + return texture + }) + .catch(function (error) { + console.error("THREE.GLTFLoader: Couldn't load texture", sourceURI) + throw error + }) + + sourceCache[sourceIndex] = promise + return promise +} + export const GLTFLoaderFunctions = { loadAccessor, loadBufferView, loadBuffer, - computeBounds + computeBounds, + loadMaterial, + assignTexture, + loadTexture, + loadImageSource, + loadTextureImage } diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 25efdd2ad6..4cec1366e6 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -53,6 +53,7 @@ import { setComponent, UndefinedEntity, useComponent, + useOptionalComponent, UUIDComponent } from '@etherealengine/ecs' import { @@ -71,7 +72,7 @@ import { import { TransformComponent } from '@etherealengine/spatial' import { useGet } from '@etherealengine/spatial/src/common/functions/FeathersHooks' import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' -import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' +import { addObjectToGroup, removeObjectFromGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' import { VisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' @@ -642,6 +643,7 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do if (!geometry.value) return const mesh = new Mesh(geometry.value as BufferGeometry, new MeshBasicMaterial()) + /** @todo multiple primitive support */ setComponent(props.entity, MeshComponent, mesh) addObjectToGroup(props.entity, mesh) @@ -655,6 +657,7 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do return () => { removeComponent(props.entity, MeshComponent) + removeObjectFromGroup(props.entity, mesh) } }, [geometry]) @@ -766,6 +769,8 @@ const PrimitiveIndicesAttributeReactor = (props: { const MaterialReactor = (props: { nodeIndex: number; documentID: string; primitiveIndex: number; entity: Entity }) => { const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const meshComponent = useOptionalComponent(props.entity, MeshComponent) + const nodes = documentState.nodes!.get(NO_PROXY)! const node = nodes[props.nodeIndex]! @@ -773,15 +778,19 @@ const MaterialReactor = (props: { nodeIndex: number; documentID: string; primiti const primitive = mesh.primitives[props.primitiveIndex] - const material = primitive.material - useEffect(() => { - // const material = GLTFLoaderFunctions.createMaterial( - // getParserOptions(props.entity), - // documentState.get(NO_PROXY) as GLTF.IGLTF, - // material - // ) - }, []) + if (typeof primitive.material !== 'number' || !meshComponent) return + + GLTFLoaderFunctions.loadMaterial( + getParserOptions(props.entity), + documentState.get(NO_PROXY) as GLTF.IGLTF, + primitive.material! + ).then((material) => { + ;(meshComponent.get(NO_PROXY) as Mesh).material = material + }) + + /** @todo use material API instead of hardcoding */ + }, [meshComponent, primitive.material]) return null } From 08e1d80303c1751a18e2c4a06d4275451a31dc80 Mon Sep 17 00:00:00 2001 From: HexaField Date: Mon, 29 Jul 2024 14:43:51 +1000 Subject: [PATCH 06/47] add kmu extension --- .../src/assets/loaders/gltf/GLTFExtensions.ts | 44 +++++++-------- packages/engine/src/gltf/GLTFExtensions.ts | 53 +++++++++++++++++-- .../engine/src/gltf/GLTFLoaderFunctions.ts | 28 ++++++---- 3 files changed, 91 insertions(+), 34 deletions(-) diff --git a/packages/engine/src/assets/loaders/gltf/GLTFExtensions.ts b/packages/engine/src/assets/loaders/gltf/GLTFExtensions.ts index 1db5d27d5d..48bffab99e 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFExtensions.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFExtensions.ts @@ -53,28 +53,28 @@ import { assignExtrasToUserData } from './GLTFLoaderFunctions' import { GLTFParser } from './GLTFParser' export const EXTENSIONS = { - KHR_BINARY_GLTF: 'KHR_binary_glTF', - KHR_DRACO_MESH_COMPRESSION: 'KHR_draco_mesh_compression', - KHR_LIGHTS_PUNCTUAL: 'KHR_lights_punctual', - KHR_MATERIALS_CLEARCOAT: 'KHR_materials_clearcoat', - KHR_MATERIALS_IOR: 'KHR_materials_ior', - KHR_MATERIALS_SHEEN: 'KHR_materials_sheen', - KHR_MATERIALS_SPECULAR: 'KHR_materials_specular', - KHR_MATERIALS_TRANSMISSION: 'KHR_materials_transmission', - KHR_MATERIALS_IRIDESCENCE: 'KHR_materials_iridescence', - KHR_MATERIALS_ANISOTROPY: 'KHR_materials_anisotropy', - KHR_MATERIALS_UNLIT: 'KHR_materials_unlit', - KHR_MATERIALS_VOLUME: 'KHR_materials_volume', - KHR_TEXTURE_BASISU: 'KHR_texture_basisu', - KHR_TEXTURE_TRANSFORM: 'KHR_texture_transform', - KHR_MESH_QUANTIZATION: 'KHR_mesh_quantization', - KHR_MATERIALS_EMISSIVE_STRENGTH: 'KHR_materials_emissive_strength', - EXT_MATERIALS_BUMP: 'EXT_materials_bump', - EXT_TEXTURE_WEBP: 'EXT_texture_webp', - EXT_TEXTURE_AVIF: 'EXT_texture_avif', - EXT_MESHOPT_COMPRESSION: 'EXT_meshopt_compression', - EXT_MESH_GPU_INSTANCING: 'EXT_mesh_gpu_instancing', - EE_MATERIAL: 'EE_material' + KHR_BINARY_GLTF: 'KHR_binary_glTF' as const, + KHR_DRACO_MESH_COMPRESSION: 'KHR_draco_mesh_compression' as const, + KHR_LIGHTS_PUNCTUAL: 'KHR_lights_punctual' as const, + KHR_MATERIALS_CLEARCOAT: 'KHR_materials_clearcoat' as const, + KHR_MATERIALS_IOR: 'KHR_materials_ior' as const, + KHR_MATERIALS_SHEEN: 'KHR_materials_sheen' as const, + KHR_MATERIALS_SPECULAR: 'KHR_materials_specular' as const, + KHR_MATERIALS_TRANSMISSION: 'KHR_materials_transmission' as const, + KHR_MATERIALS_IRIDESCENCE: 'KHR_materials_iridescence' as const, + KHR_MATERIALS_ANISOTROPY: 'KHR_materials_anisotropy' as const, + KHR_MATERIALS_UNLIT: 'KHR_materials_unlit' as const, + KHR_MATERIALS_VOLUME: 'KHR_materials_volume' as const, + KHR_TEXTURE_BASISU: 'KHR_texture_basisu' as const, + KHR_TEXTURE_TRANSFORM: 'KHR_texture_transform' as const, + KHR_MESH_QUANTIZATION: 'KHR_mesh_quantization' as const, + KHR_MATERIALS_EMISSIVE_STRENGTH: 'KHR_materials_emissive_strength' as const, + EXT_MATERIALS_BUMP: 'EXT_materials_bump' as const, + EXT_TEXTURE_WEBP: 'EXT_texture_webp' as const, + EXT_TEXTURE_AVIF: 'EXT_texture_avif' as const, + EXT_MESHOPT_COMPRESSION: 'EXT_meshopt_compression' as const, + EXT_MESH_GPU_INSTANCING: 'EXT_mesh_gpu_instancing' as const, + EE_MATERIAL: 'EE_material' as const } /** diff --git a/packages/engine/src/gltf/GLTFExtensions.ts b/packages/engine/src/gltf/GLTFExtensions.ts index a3b79300d2..81dce835a4 100644 --- a/packages/engine/src/gltf/GLTFExtensions.ts +++ b/packages/engine/src/gltf/GLTFExtensions.ts @@ -25,14 +25,21 @@ Ethereal Engine. All Rights Reserved. import { getState } from '@etherealengine/hyperflux' import { GLTF } from '@gltf-transform/core' -import { BufferGeometry, NormalBufferAttributes } from 'three' +import { + BufferGeometry, + Color, + LinearSRGBColorSpace, + MeshBasicMaterial, + NormalBufferAttributes, + SRGBColorSpace +} from 'three' import { ATTRIBUTES, WEBGL_COMPONENT_TYPES } from '../assets/loaders/gltf/GLTFConstants' import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' import { AssetLoaderState } from '../assets/state/AssetLoaderState' import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' -const khr_draco_mesh_compression = { +const KHR_DRACO_MESH_COMPRESSION = { decodePrimitive(options: GLTFParserOptions, json: GLTF.IGLTF, primitive: GLTF.IMeshPrimitive) { const dracoLoader = getState(AssetLoaderState).gltfLoader.dracoLoader! const dracoMeshCompressionExtension = primitive.extensions![EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] as any @@ -83,6 +90,46 @@ const khr_draco_mesh_compression = { } } +const KHR_MATERIALS_UNLIT = { + getMaterialType() { + return MeshBasicMaterial + }, + + extendParams(options: GLTFParserOptions, json: GLTF.IGLTF, materialParams, materialDef) { + const pending = [] as Promise[] + + materialParams.color = new Color(1.0, 1.0, 1.0) + materialParams.opacity = 1.0 + + const metallicRoughness = materialDef.pbrMetallicRoughness + + if (metallicRoughness) { + if (Array.isArray(metallicRoughness.baseColorFactor)) { + const array = metallicRoughness.baseColorFactor + + materialParams.color.setRGB(array[0], array[1], array[2], LinearSRGBColorSpace) + materialParams.opacity = array[3] + } + + if (metallicRoughness.baseColorTexture !== undefined) { + pending.push( + GLTFLoaderFunctions.assignTexture( + options, + json, + materialParams, + 'map', + metallicRoughness.baseColorTexture, + SRGBColorSpace + ) + ) + } + } + + return Promise.all(pending) + } +} + export const GLTFExtensions = { - [EXTENSIONS.KHR_DRACO_MESH_COMPRESSION]: khr_draco_mesh_compression + [EXTENSIONS.KHR_DRACO_MESH_COMPRESSION]: KHR_DRACO_MESH_COMPRESSION, + [EXTENSIONS.KHR_MATERIALS_UNLIT]: KHR_MATERIALS_UNLIT } diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 179db79dae..b8874b0701 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -31,6 +31,7 @@ import { Color, DoubleSide, ImageBitmapLoader, + ImageLoader, InterleavedBuffer, InterleavedBufferAttribute, LinearFilter, @@ -79,7 +80,7 @@ const loadAccessor = (options: GLTFParserOptions, json: GLTF.IGLTF, accessorInde return Promise.resolve(new BufferAttribute(array, itemSize, normalized)) } - const pendingBufferViews = [] as Array | null> // todo + const pendingBufferViews = [] as Array | null> if (accessorDef.bufferView !== undefined) { pendingBufferViews.push(GLTFLoaderFunctions.loadBufferView(options, json, accessorDef.bufferView)) @@ -318,17 +319,17 @@ const loadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialInde const materialParams = {} as any // todo const materialExtensions = materialDef.extensions || {} - const pending = [] as any[] // todo + const pending = [] as Array> if (!materialExtensions[EXTENSIONS.EE_MATERIAL] && materialExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT]) { - /** @todo */ - // const kmuExtension = extensions[EXTENSIONS.KHR_MATERIALS_UNLIT] - // materialType = kmuExtension.getMaterialType() - // pending.push(kmuExtension.extendParams(materialParams, materialDef, parser)) + const kmuExtension = GLTFExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT] + materialType = kmuExtension.getMaterialType() + pending.push(kmuExtension.extendParams(options, json, materialParams, materialDef)) } else { // Specification: // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#metallic-roughness-material + /** @todo move this to a base plugin */ const metallicRoughness = materialDef.pbrMetallicRoughness || {} materialParams.color = new Color(1.0, 1.0, 1.0) @@ -381,6 +382,10 @@ const loadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialInde /** @todo expose 'getMaterialType' API */ materialType = MeshStandardMaterial + // materialType = this._invokeOne(function (ext) { + // return ext.getMaterialType && ext.getMaterialType(materialIndex) + // }) + /** @todo expose API */ // pending.push( // Promise.all( @@ -619,7 +624,12 @@ const loadTextureImage = ( const sourceCache = {} as any // todo -const loadImageSource = (options: GLTFParserOptions, json: GLTF.IGLTF, sourceIndex, loader) => { +const loadImageSource = ( + options: GLTFParserOptions, + json: GLTF.IGLTF, + sourceIndex: number, + loader: ImageLoader | ImageBitmapLoader +) => { // eslint-disable-next-line @typescript-eslint/no-this-alias if (sourceCache[sourceIndex] !== undefined) { @@ -651,7 +661,7 @@ const loadImageSource = (options: GLTFParserOptions, json: GLTF.IGLTF, sourceInd return new Promise(function (resolve, reject) { let onLoad = resolve - if (loader.isImageBitmapLoader === true) { + if ((loader as ImageBitmapLoader).isImageBitmapLoader === true) { onLoad = function (imageBitmap) { const texture = new Texture(imageBitmap) texture.needsUpdate = true @@ -663,7 +673,7 @@ const loadImageSource = (options: GLTFParserOptions, json: GLTF.IGLTF, sourceInd loader.load(LoaderUtils.resolveURL(sourceURI, options.path), onLoad, undefined, reject) }) }) - .then(function (texture) { + .then(function (texture: Texture) { // Clean up resources and configure Texture. if (isObjectURL === true) { From c9ae4c484f8b2831b2cc5124a96147f3bec96560 Mon Sep 17 00:00:00 2001 From: HexaField Date: Wed, 31 Jul 2024 12:26:32 +1000 Subject: [PATCH 07/47] load material definitions separate to instances --- packages/engine/src/gltf/GLTFState.tsx | 116 +++++++++++++++++++++---- 1 file changed, 99 insertions(+), 17 deletions(-) diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 64ba58ced9..7eb2ebf412 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -46,6 +46,7 @@ import { Entity, EntityUUID, getComponent, + getMutableComponent, getOptionalComponent, hasComponent, removeComponent, @@ -53,7 +54,6 @@ import { setComponent, UndefinedEntity, useComponent, - useOptionalComponent, UUIDComponent } from '@etherealengine/ecs' import { @@ -84,6 +84,10 @@ import { import { EngineState } from '@etherealengine/spatial/src/EngineState' import { Physics } from '@etherealengine/spatial/src/physics/classes/Physics' import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' +import { + MaterialInstanceComponent, + MaterialStateComponent +} from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { ATTRIBUTES } from '../assets/loaders/gltf/GLTFConstants' import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' import { assignExtrasToUserData } from '../assets/loaders/gltf/GLTFLoaderFunctions' @@ -409,7 +413,8 @@ const ChildGLTFReactor = (props: { source: string }) => { export const DocumentReactor = (props: { documentID: string; parentUUID: EntityUUID }) => { const nodeState = useHookstate(getMutableState(GLTFNodeState)[props.documentID]) - if (!nodeState.value) return null + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + if (!documentState.value || !nodeState.value) return null return ( <> {Object.entries(nodeState.get(NO_PROXY)).map(([uuid, { nodeIndex, childIndex, parentUUID }]) => ( @@ -421,10 +426,88 @@ export const DocumentReactor = (props: { documentID: string; parentUUID: EntityU documentID={props.documentID} /> ))} + {documentState + .get(NO_PROXY) + .materials?.map((material, index) => ( + + ))} ) } +const MaterialReactor = (props: { index: number; parentUUID: EntityUUID; documentID: string }) => { + const documentState = useMutableState(GLTFDocumentState)[props.documentID] + const materials = documentState.materials! + + const material = materials[props.index]!.get(NO_PROXY) as GLTF.IMaterial + + const parentEntity = UUIDComponent.useEntityByUUID(props.parentUUID) + + const entityState = useHookstate(UndefinedEntity) + const entity = entityState.value + + useEffect(() => { + const uuid = (props.documentID + '-material-' + props.index) as EntityUUID + const entity = UUIDComponent.getOrCreateEntityByUUID(uuid) + + setComponent(entity, UUIDComponent, uuid) + setComponent(entity, SourceComponent, props.documentID) + + /** Ensure all base components are added for synchronous mount */ + setComponent(entity, EntityTreeComponent, { parentEntity, childIndex: props.index }) + setComponent(entity, NameComponent, material.name ?? 'Material-' + props.index) + + entityState.set(entity) + + return () => { + //check if entity is in some other document + if (hasComponent(entity, UUIDComponent)) { + const uuid = getComponent(entity, UUIDComponent) + const documents = getState(GLTFDocumentState) + for (const documentID in documents) { + const document = documents[documentID] + if (!document?.materials) continue + for (const material of document.materials) { + if (material.extensions?.[UUIDComponent.jsonID] === uuid) return + } + } + } + removeEntity(entity) + } + }, []) + + useLayoutEffect(() => { + if (!entity) return + + setComponent(entity, EntityTreeComponent, { parentEntity, childIndex: props.index }) + }, [entity, parentEntity, props.index]) + + useLayoutEffect(() => { + if (!entity) return + + setComponent(entity, NameComponent, material.name ?? 'Material-' + props.index) + }, [entity, material.name]) + + useLayoutEffect(() => { + if (!entity) return + + GLTFLoaderFunctions.loadMaterial( + getParserOptions(entity), + documentState.get(NO_PROXY) as GLTF.IGLTF, + props.index + ).then((material) => { + setComponent(entity, MaterialStateComponent, { material }) + }) + }, [entity, material]) + + return null +} + const ParentNodeReactor = (props: { nodeIndex: number childIndex: number @@ -704,7 +787,7 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do return ( <> {typeof primitive.material === 'number' && ( - { +const MaterialInstanceReactor = (props: { + nodeIndex: number + documentID: string + primitiveIndex: number + entity: Entity +}) => { const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) - const meshComponent = useOptionalComponent(props.entity, MeshComponent) - const nodes = documentState.nodes!.get(NO_PROXY)! const node = nodes[props.nodeIndex]! @@ -816,19 +902,15 @@ const MaterialReactor = (props: { nodeIndex: number; documentID: string; primiti const primitive = mesh.primitives[props.primitiveIndex] - useEffect(() => { - if (typeof primitive.material !== 'number' || !meshComponent) return + const materialUUID = (props.documentID + '-material-' + primitive.material!) as EntityUUID + const materialEntity = UUIDComponent.useEntityByUUID(materialUUID) - GLTFLoaderFunctions.loadMaterial( - getParserOptions(props.entity), - documentState.get(NO_PROXY) as GLTF.IGLTF, - primitive.material! - ).then((material) => { - ;(meshComponent.get(NO_PROXY) as Mesh).material = material - }) + useEffect(() => { + if (typeof primitive.material !== 'number' || !materialEntity) return - /** @todo use material API instead of hardcoding */ - }, [meshComponent, primitive.material]) + setComponent(props.entity, MaterialInstanceComponent) + getMutableComponent(props.entity, MaterialInstanceComponent).uuid.merge([materialUUID]) + }, [materialEntity, primitive.material]) return null } From 248af55aa2bd720e3191fb314ac91cc8632fad61 Mon Sep 17 00:00:00 2001 From: HexaField Date: Thu, 1 Aug 2024 12:56:27 +1000 Subject: [PATCH 08/47] turn functions into hooks --- .../src/assets/loaders/gltf/GLTFParser.ts | 4 +- .../engine/src/gltf/GLTFLoaderFunctions.ts | 753 ++++++++++-------- packages/engine/src/gltf/GLTFState.tsx | 120 +-- 3 files changed, 496 insertions(+), 381 deletions(-) diff --git a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts index 60651b0c97..aa20ed3078 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts @@ -127,7 +127,7 @@ export type GLTFParserOptions = { body: null | ArrayBuffer crossOrigin: 'anonymous' | string ktx2Loader: KTX2Loader - manager: LoadingManager | any + manager: LoadingManager meshoptDecoder: any path: string requestHeader: Record @@ -689,7 +689,7 @@ export class GLTFParser { if (sourceDef.uri) { const handler = options.manager.getHandler(sourceDef.uri) - if (handler !== null) loader = handler + if (handler !== null) loader = handler as any } return this.loadTextureImage(textureIndex, sourceIndex, loader) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index b8874b0701..70dac3a65c 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -23,13 +23,16 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ +import { NO_PROXY, useHookstate } from '@etherealengine/hyperflux' import { GLTF } from '@gltf-transform/core' +import { useEffect } from 'react' import { Box3, BufferAttribute, BufferGeometry, Color, DoubleSide, + FrontSide, ImageBitmapLoader, ImageLoader, InterleavedBuffer, @@ -45,7 +48,6 @@ import { Sphere, Texture, TextureLoader, - Vector2, Vector3 } from 'three' import { FileLoader } from '../assets/loaders/base/FileLoader' @@ -68,33 +70,41 @@ import { GLTFExtensions } from './GLTFExtensions' // todo make this a state const cache = new GLTFRegistry() -const loadAccessor = (options: GLTFParserOptions, json: GLTF.IGLTF, accessorIndex: number) => { - const accessorDef = json.accessors![accessorIndex] +const useLoadAccessor = (options: GLTFParserOptions, json: GLTF.IGLTF, accessorIndex?: number) => { + const result = useHookstate(null) - if (accessorDef.bufferView === undefined && accessorDef.sparse === undefined) { - const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] - const TypedArray = WEBGL_COMPONENT_TYPES[accessorDef.componentType] - const normalized = accessorDef.normalized === true + const accessorDef = typeof accessorIndex === 'number' ? json.accessors![accessorIndex] : null - const array = new TypedArray(accessorDef.count * itemSize) - return Promise.resolve(new BufferAttribute(array, itemSize, normalized)) - } + const bufferView = GLTFLoaderFunctions.useLoadBufferView(options, json, accessorDef?.bufferView) - const pendingBufferViews = [] as Array | null> + const sparseBufferViewIndices = GLTFLoaderFunctions.useLoadBufferView( + options, + json, + accessorDef?.sparse?.indices?.bufferView + ) + const sparseBufferViewValues = GLTFLoaderFunctions.useLoadBufferView( + options, + json, + accessorDef?.sparse?.values?.bufferView + ) - if (accessorDef.bufferView !== undefined) { - pendingBufferViews.push(GLTFLoaderFunctions.loadBufferView(options, json, accessorDef.bufferView)) - } else { - pendingBufferViews.push(null) - } + useEffect(() => { + if (!accessorDef) return - if (accessorDef.sparse !== undefined) { - pendingBufferViews.push(GLTFLoaderFunctions.loadBufferView(options, json, accessorDef.sparse.indices.bufferView)) - pendingBufferViews.push(GLTFLoaderFunctions.loadBufferView(options, json, accessorDef.sparse.values.bufferView)) - } + if (accessorDef.bufferView === undefined && accessorDef.sparse === undefined) { + const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] + const TypedArray = WEBGL_COMPONENT_TYPES[accessorDef.componentType] + const normalized = accessorDef.normalized === true + + const array = new TypedArray(accessorDef.count * itemSize) + result.set(new BufferAttribute(array, itemSize, normalized)) + return + } + + if (accessorDef.bufferView && !bufferView) return + if (accessorDef.sparse && !sparseBufferViewIndices && !sparseBufferViewValues) return - return Promise.all(pendingBufferViews).then(function (bufferViews) { - const bufferView = bufferViews[0] + const bufferViews = accessorDef.bufferView ? [bufferView, null] : [sparseBufferViewIndices, sparseBufferViewValues] const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] const TypedArray = WEBGL_COMPONENT_TYPES[accessorDef.componentType] @@ -184,45 +194,73 @@ const loadAccessor = (options: GLTFParserOptions, json: GLTF.IGLTF, accessorInde } } - return bufferAttribute - }) + result.set(bufferAttribute) + }, [bufferView, sparseBufferViewIndices, sparseBufferViewValues]) + + return result.value } -const loadBufferView = async (options: GLTFParserOptions, json: GLTF.IGLTF, bufferViewIndex: number) => { - const bufferViewDef = json.bufferViews![bufferViewIndex] +const useLoadBufferView = (options: GLTFParserOptions, json: GLTF.IGLTF, bufferViewIndex?: number) => { + const result = useHookstate(null) + + const bufferViewDef = typeof bufferViewIndex === 'number' ? json.bufferViews![bufferViewIndex] : null + const buffer = GLTFLoaderFunctions.useLoadBuffer(options, json, bufferViewDef?.buffer) - const buffer = await GLTFLoaderFunctions.loadBuffer(options, json, bufferViewDef.buffer) + useEffect(() => { + if (!bufferViewDef || !buffer) return - const byteLength = bufferViewDef.byteLength || 0 - const byteOffset = bufferViewDef.byteOffset || 0 + const byteLength = bufferViewDef.byteLength || 0 + const byteOffset = bufferViewDef.byteOffset || 0 - return buffer.slice(byteOffset, byteOffset + byteLength) + result.set(buffer.slice(byteOffset, byteOffset + byteLength)) + }, [buffer]) + + return result.value } -const loadBuffer = async (options: GLTFParserOptions, json: GLTF.IGLTF, bufferIndex: number) => { - const bufferDef = json.buffers![bufferIndex] +const useLoadBuffer = (options: GLTFParserOptions, json: GLTF.IGLTF, bufferIndex?: number) => { + const result = useHookstate(null) - if (bufferDef.type && bufferDef.type !== 'arraybuffer') { - throw new Error('THREE.GLTFLoader: ' + bufferDef.type + ' buffer type is not supported.') - } + const bufferDef = typeof bufferIndex === 'number' ? json.buffers![bufferIndex] : null - // If present, GLB container is required to be the first buffer. - if (bufferDef.uri === undefined && bufferIndex === 0) { - return Promise.resolve(options.body!) - } + useEffect(() => { + if (!bufferDef) return - /** @todo use a global file loader */ - const fileLoader = new FileLoader(options.manager) - fileLoader.setResponseType('arraybuffer') - if (options.crossOrigin === 'use-credentials') { - fileLoader.setWithCredentials(true) - } + if (bufferDef.type && bufferDef.type !== 'arraybuffer') { + console.warn('THREE.GLTFLoader: ' + bufferDef.type + ' buffer type is not supported.') + } + }, [bufferDef?.type]) - return new Promise(function (resolve, reject) { - fileLoader.load(LoaderUtils.resolveURL(bufferDef.uri!, options.path), resolve as any, undefined, function () { - reject(new Error('THREE.GLTFLoader: Failed to load buffer "' + bufferDef.uri + '".')) - }) - }) + useEffect(() => { + if (!bufferDef) return + + // If present, GLB container is required to be the first buffer. + if (bufferDef.uri === undefined && bufferIndex === 0) { + result.set(options.body!) + return + } + + /** @todo use a global file loader */ + const fileLoader = new FileLoader(options.manager) + fileLoader.setResponseType('arraybuffer') + if (options.crossOrigin === 'use-credentials') { + fileLoader.setWithCredentials(true) + } + + fileLoader.load( + LoaderUtils.resolveURL(bufferDef.uri!, options.path), + (val: ArrayBuffer) => { + result.set(val) + }, + undefined, + function () { + result.set(null) + console.error(new Error('THREE.GLTFLoader: Failed to load buffer "' + bufferDef.uri + '".')) + } + ) + }, [bufferDef?.uri]) + + return result.value } export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primitiveDef: GLTF.IMeshPrimitive) { @@ -312,172 +350,193 @@ export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primit * @param {number} materialIndex * @return {Promise} */ -const loadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialIndex: number) => { - const materialDef = json.materials![materialIndex] +const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialIndex: number) => { + const materialType = useHookstate<'standard' | 'basic'>('standard') - let materialType - const materialParams = {} as any // todo - const materialExtensions = materialDef.extensions || {} - - const pending = [] as Array> - - if (!materialExtensions[EXTENSIONS.EE_MATERIAL] && materialExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT]) { - const kmuExtension = GLTFExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT] - materialType = kmuExtension.getMaterialType() - pending.push(kmuExtension.extendParams(options, json, materialParams, materialDef)) - } else { - // Specification: - // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#metallic-roughness-material + const result = useHookstate(null as null | MeshStandardMaterial | MeshBasicMaterial) - /** @todo move this to a base plugin */ - const metallicRoughness = materialDef.pbrMetallicRoughness || {} + useEffect(() => { + const materialTypeValue = materialType.get(NO_PROXY) === 'standard' ? MeshStandardMaterial : MeshBasicMaterial + const material = new materialTypeValue() - materialParams.color = new Color(1.0, 1.0, 1.0) - materialParams.opacity = 1.0 + result.set(material) - if (Array.isArray(metallicRoughness.baseColorFactor)) { - const array = metallicRoughness.baseColorFactor - - materialParams.color.setRGB(array[0], array[1], array[2], LinearSRGBColorSpace) - materialParams.opacity = array[3] - } + assignExtrasToUserData(material, materialDef) - if (metallicRoughness.baseColorTexture !== undefined) { - pending.push( - GLTFLoaderFunctions.assignTexture( - options, - json, - materialParams, - 'map', - metallicRoughness.baseColorTexture, - SRGBColorSpace - ) - ) - } + /** @todo */ + // parser.associations.set(material, { materials: materialIndex }) - materialParams.metalness = metallicRoughness.metallicFactor !== undefined ? metallicRoughness.metallicFactor : 1.0 - materialParams.roughness = metallicRoughness.roughnessFactor !== undefined ? metallicRoughness.roughnessFactor : 1.0 - - if (metallicRoughness.metallicRoughnessTexture !== undefined) { - pending.push( - GLTFLoaderFunctions.assignTexture( - options, - json, - materialParams, - 'metalnessMap', - metallicRoughness.metallicRoughnessTexture - ) - ) - pending.push( - GLTFLoaderFunctions.assignTexture( - options, - json, - materialParams, - 'roughnessMap', - metallicRoughness.metallicRoughnessTexture - ) - ) - } + if (materialDef.extensions) addUnknownExtensionsToUserData(GLTFExtensions, material, materialDef) - /** @todo expose 'getMaterialType' API */ - materialType = MeshStandardMaterial - - // materialType = this._invokeOne(function (ext) { - // return ext.getMaterialType && ext.getMaterialType(materialIndex) - // }) - - /** @todo expose API */ - // pending.push( - // Promise.all( - // this._invokeAll(function (ext) { - // return ext.extendMaterialParams && ext.extendMaterialParams(materialIndex, materialParams) - // }) - // ) - // ) - } + result.set(material) + }, [materialType]) - if (materialDef.doubleSided === true) { - materialParams.side = DoubleSide - } + const materialDef = json.materials![materialIndex] - const alphaMode = materialDef.alphaMode || ALPHA_MODES.OPAQUE + const materialExtensions = materialDef.extensions || {} - if (alphaMode === ALPHA_MODES.BLEND) { - materialParams.transparent = true + /** @todo expose 'getMaterialType' API */ + + // materialType = this._invokeOne(function (ext) { + // return ext.getMaterialType && ext.getMaterialType(materialIndex) + // }) + + /** @todo expose API */ + // pending.push( + // Promise.all( + // this._invokeAll(function (ext) { + // return ext.extendMaterialParams && ext.extendMaterialParams(materialIndex, materialParams) + // }) + // ) + // ) + // } + + // if (!materialExtensions[EXTENSIONS.EE_MATERIAL] && materialExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT]) { + // const kmuExtension = GLTFExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT] + // materialType = kmuExtension.getMaterialType() + // pending.push(kmuExtension.extendParams(options, json, materialParams, materialDef)) + // } else { + // Specification: + // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#metallic-roughness-material + + const map = GLTFLoaderFunctions.assignTexture(options, json, materialDef.pbrMetallicRoughness?.baseColorTexture) + + useEffect(() => { + if (!map) return + map.colorSpace = SRGBColorSpace + result.value?.setValues({ map }) + }, [map]) + + useEffect(() => { + if (Array.isArray(materialDef.pbrMetallicRoughness?.baseColorFactor)) { + const array = materialDef.pbrMetallicRoughness.baseColorFactor + result.value?.setValues({ + color: new Color().setRGB(array[0], array[1], array[2], LinearSRGBColorSpace), + opacity: array[3] + }) + } + }, [materialDef.pbrMetallicRoughness?.baseColorFactor]) + + useEffect(() => { + result.value?.setValues({ + metalness: + materialDef.pbrMetallicRoughness?.metallicFactor !== undefined + ? materialDef.pbrMetallicRoughness.metallicFactor + : 1.0 + }) + }, [materialDef.pbrMetallicRoughness?.metallicFactor]) + + useEffect(() => { + result.value?.setValues({ + roughness: + materialDef.pbrMetallicRoughness?.roughnessFactor !== undefined + ? materialDef.pbrMetallicRoughness.roughnessFactor + : 1.0 + }) + }, [materialDef.pbrMetallicRoughness?.roughnessFactor]) + + const metalnessMap = GLTFLoaderFunctions.assignTexture( + options, + json, + materialType.value === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture + ) + + useEffect(() => { + if (!metalnessMap) return + result.value?.setValues({ metalnessMap }) + }, [metalnessMap]) + + const roughnessMap = GLTFLoaderFunctions.assignTexture( + options, + json, + materialType.value === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture + ) + + useEffect(() => { + if (!roughnessMap) return + result.value?.setValues({ roughnessMap }) + }, [roughnessMap]) + + useEffect(() => { + result.value?.setValues({ side: materialDef.doubleSided === true ? DoubleSide : FrontSide }) + }, [materialDef.doubleSided]) + + useEffect(() => { + const alphaMode = materialDef.alphaMode || ALPHA_MODES.OPAQUE + result.value?.setValues({ transparent: alphaMode !== ALPHA_MODES.OPAQUE }) // See: https://github.com/mrdoob/three.js/issues/17706 - if (materialParams.depthWrite === undefined) { - materialParams.depthWrite = false + if (alphaMode !== ALPHA_MODES.OPAQUE) { + result.value?.setValues({ depthWrite: false }) } - } else { - materialParams.transparent = false + }, [materialDef.alphaMode]) - if (alphaMode === ALPHA_MODES.MASK) { - materialParams.alphaTest = materialDef.alphaCutoff !== undefined ? materialDef.alphaCutoff : 0.5 + useEffect(() => { + if (materialDef.alphaMode === ALPHA_MODES.MASK) { + result.value?.setValues({ alphaTest: materialDef.alphaCutoff !== undefined ? materialDef.alphaCutoff : 0.5 }) + } else { + result.value?.setValues({ alphaTest: 0 }) } - } + }, [materialDef.alphaCutoff]) - if (materialDef.normalTexture !== undefined && materialType !== MeshBasicMaterial) { - pending.push( - GLTFLoaderFunctions.assignTexture(options, json, materialParams, 'normalMap', materialDef.normalTexture) - ) + const normalMap = GLTFLoaderFunctions.assignTexture( + options, + json, + materialType.value === 'basic' ? undefined : materialDef.normalTexture + ) - materialParams.normalScale = new Vector2(1, 1) + useEffect(() => { + if (!normalMap) return + result.value?.setValues({ normalMap }) + }, [normalMap]) - if (materialDef.normalTexture.scale !== undefined) { - const scale = materialDef.normalTexture.scale + // useEffect(() => { + // materialParams.normalScale = new Vector2(1, 1) - materialParams.normalScale.set(scale, scale) - } - } + // if (materialDef.normalTexture.scale !== undefined) { + // const scale = materialDef.normalTexture.scale - if (materialDef.occlusionTexture !== undefined && materialType !== MeshBasicMaterial) { - pending.push( - GLTFLoaderFunctions.assignTexture(options, json, materialParams, 'aoMap', materialDef.occlusionTexture) - ) + // materialParams.normalScale.set(scale, scale) + // } + // }, []) - if (materialDef.occlusionTexture.strength !== undefined) { - materialParams.aoMapIntensity = materialDef.occlusionTexture.strength - } - } + const aoMap = GLTFLoaderFunctions.assignTexture( + options, + json, + materialType.value === 'basic' ? undefined : materialDef.occlusionTexture + ) - if (materialDef.emissiveFactor !== undefined && materialType !== MeshBasicMaterial) { - const emissiveFactor = materialDef.emissiveFactor - materialParams.emissive = new Color().setRGB( - emissiveFactor[0], - emissiveFactor[1], - emissiveFactor[2], - LinearSRGBColorSpace - ) - } + useEffect(() => { + if (!aoMap) return + result.value?.setValues({ aoMap }) + }, [aoMap]) - if (materialDef.emissiveTexture !== undefined && materialType !== MeshBasicMaterial) { - pending.push( - GLTFLoaderFunctions.assignTexture( - options, - json, - materialParams, - 'emissiveMap', - materialDef.emissiveTexture, - SRGBColorSpace - ) - ) - } + useEffect(() => { + result.value?.setValues({ aoMapIntensity: materialDef.occlusionTexture?.strength ?? 1.0 }) + }, [materialDef.occlusionTexture?.strength]) - return Promise.all(pending).then(function () { - const material = new materialType(materialParams) + useEffect(() => { + const emissiveFactor = materialDef.emissiveFactor + if (!emissiveFactor) return - if (materialDef.name) material.name = materialDef.name + result.value?.setValues({ + emissive: new Color().setRGB(emissiveFactor[0], emissiveFactor[1], emissiveFactor[2], LinearSRGBColorSpace) + }) + }, [materialDef.emissiveFactor]) - assignExtrasToUserData(material, materialDef) + const emissiveMap = GLTFLoaderFunctions.assignTexture( + options, + json, + materialType.value === 'basic' ? undefined : materialDef.emissiveTexture + ) - /** @todo */ - // parser.associations.set(material, { materials: materialIndex }) + useEffect(() => { + if (!emissiveMap) return + emissiveMap.colorSpace = SRGBColorSpace + result.value?.setValues({ emissiveMap }) + }, [emissiveMap]) - if (materialDef.extensions) addUnknownExtensionsToUserData(GLTFExtensions, material, materialDef) - - return material - }) + return result.value as MeshStandardMaterial | null } /** @@ -487,15 +546,23 @@ const loadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialInde * @param {Object} mapDef * @return {Promise} */ -const assignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, materialParams, mapName, mapDef, colorSpace?) => { - // eslint-disable-next-line @typescript-eslint/no-this-alias +const assignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, mapDef?: GLTF.ITextureInfo) => { + const result = useHookstate(null) + + const texture = GLTFLoaderFunctions.useLoadTexture(options, json, mapDef?.index) + + useEffect(() => { + if (!texture) { + result.set(null) + return + } - return GLTFLoaderFunctions.loadTexture(options, json, mapDef.index).then(function (texture) { - if (!texture) return null + if (!mapDef) return if (mapDef.texCoord !== undefined && mapDef.texCoord > 0) { - texture = texture.clone() - texture.channel = mapDef.texCoord + const textureClone = texture.clone() + textureClone.channel = mapDef.texCoord + result.set(textureClone) } if (GLTFExtensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM]) { @@ -509,15 +576,9 @@ const assignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, materialPar // parser.associations.set(texture, gltfReference) } } + }, [texture, mapDef]) - if (colorSpace !== undefined) { - texture.colorSpace = colorSpace - } - - materialParams[mapName] = texture - - return texture - }) + return result.value as Texture | null } /** @@ -525,184 +586,220 @@ const assignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, materialPar * @param {number} textureIndex * @return {Promise} */ -const loadTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, textureIndex: number) => { - const textureDef = json.textures![textureIndex] - const sourceIndex = textureDef.source! - const sourceDef = json.images![sourceIndex] - - let isSafari = false - let isFirefox = false - let firefoxVersion = -1 as any // ??? - - if (typeof navigator !== 'undefined') { - isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent) === true - isFirefox = navigator.userAgent.indexOf('Firefox') > -1 - firefoxVersion = isFirefox ? navigator.userAgent.match(/Firefox\/([0-9]+)\./)![1] : -1 - } +const useLoadTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, textureIndex?: number) => { + const result = useHookstate(null) + + const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null + const sourceIndex = textureDef?.source! + const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null + + const textureLoader = useHookstate(() => { + let isSafari = false + let isFirefox = false + let firefoxVersion = -1 as any // ??? + + if (typeof navigator !== 'undefined') { + isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent) === true + isFirefox = navigator.userAgent.indexOf('Firefox') > -1 + firefoxVersion = isFirefox ? navigator.userAgent.match(/Firefox\/([0-9]+)\./)![1] : -1 + } - let textureLoader + let textureLoader - /** @todo make global loader */ - if (typeof createImageBitmap === 'undefined' || isSafari || (isFirefox && firefoxVersion < 98)) { - textureLoader = new TextureLoader(options.manager) - } else { - textureLoader = new ImageBitmapLoader(options.manager) - } + /** @todo make global loader */ + if (typeof createImageBitmap === 'undefined' || isSafari || (isFirefox && firefoxVersion < 98)) { + textureLoader = new TextureLoader(options.manager) + } else { + textureLoader = new ImageBitmapLoader(options.manager) + } - textureLoader.setCrossOrigin(options.crossOrigin) - textureLoader.setRequestHeader(options.requestHeader) + return textureLoader + }) - let loader = textureLoader + /** @todo clean all this up */ - if (sourceDef.uri) { + textureLoader.value.setCrossOrigin(options.crossOrigin) + textureLoader.value.setRequestHeader(options.requestHeader) + + let loader = textureLoader.value + + if (sourceDef?.uri) { const handler = options.manager.getHandler(sourceDef.uri) if (handler !== null) loader = handler } - return GLTFLoaderFunctions.loadTextureImage(options, json, textureIndex, sourceIndex, loader) + const texture = GLTFLoaderFunctions.useLoadTextureImage(options, json, textureIndex, sourceIndex, loader) + + useEffect(() => { + result.set(texture) + }, [texture]) + + return result.value as Texture | null } const textureCache = {} as any // todo -const loadTextureImage = ( +const useLoadTextureImage = ( options: GLTFParserOptions, json: GLTF.IGLTF, - textureIndex: number, - sourceIndex: number, - loader + textureIndex?: number, + sourceIndex?: number, + loader?: ImageLoader | ImageBitmapLoader ) => { - // eslint-disable-next-line @typescript-eslint/no-this-alias + const result = useHookstate(null) - const textureDef = json.textures![textureIndex] - const sourceDef = json.images![sourceIndex] + const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null + const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null - const cacheKey = (sourceDef.uri || sourceDef.bufferView) + ':' + textureDef.sampler + /** @todo cache */ + // const cacheKey = (sourceDef.uri || sourceDef.bufferView) + ':' + textureDef.sampler - if (textureCache[cacheKey]) { - // See https://github.com/mrdoob/three.js/issues/21559. - return textureCache[cacheKey] - } + // if (textureCache[cacheKey]) { + // // See https://github.com/mrdoob/three.js/issues/21559. + // return textureCache[cacheKey] + // } - const promise = GLTFLoaderFunctions.loadImageSource(options, json, sourceIndex, loader) - .then(function (texture) { - texture.flipY = false + const texture = GLTFLoaderFunctions.useLoadImageSource(options, json, sourceIndex, loader) - texture.name = textureDef.name || sourceDef.name || '' + useEffect(() => { + if (!texture || !sourceDef || !textureDef) return - if ( - texture.name === '' && - typeof sourceDef.uri === 'string' && - sourceDef.uri.startsWith('data:image/') === false - ) { - texture.name = sourceDef.uri - } + texture.flipY = false - const samplers = json.samplers || {} - const sampler = samplers[textureDef.sampler!] || {} + texture.name = textureDef.name || sourceDef.name || '' - texture.magFilter = WEBGL_FILTERS[sampler.magFilter] || LinearFilter - texture.minFilter = WEBGL_FILTERS[sampler.minFilter] || LinearMipmapLinearFilter - texture.wrapS = WEBGL_WRAPPINGS[sampler.wrapS] || RepeatWrapping - texture.wrapT = WEBGL_WRAPPINGS[sampler.wrapT] || RepeatWrapping + if (texture.name === '' && typeof sourceDef.uri === 'string' && sourceDef.uri.startsWith('data:image/') === false) { + texture.name = sourceDef.uri + } - /** @todo */ - // parser.associations.set(texture, { textures: textureIndex }) + const samplers = json.samplers || {} + const sampler = samplers[textureDef.sampler!] || {} - return texture - }) - .catch(function (error) { - console.error('THREE.GLTFLoader: Error in texture onLoad for texture', sourceDef) - console.error(error) - throw error - return null - }) + texture.magFilter = WEBGL_FILTERS[sampler.magFilter] || LinearFilter + texture.minFilter = WEBGL_FILTERS[sampler.minFilter] || LinearMipmapLinearFilter + texture.wrapS = WEBGL_WRAPPINGS[sampler.wrapS] || RepeatWrapping + texture.wrapT = WEBGL_WRAPPINGS[sampler.wrapT] || RepeatWrapping + + /** @todo */ + // parser.associations.set(texture, { textures: textureIndex }) - textureCache[cacheKey] = promise + result.set(texture) + }, [textureDef, sourceDef, texture]) - return promise + // textureCache[cacheKey] = promise + + return result.value as Texture | null } -const sourceCache = {} as any // todo +// const sourceCache = {} as any // todo + +const URL = self.URL || self.webkitURL -const loadImageSource = ( +const useLoadImageSource = ( options: GLTFParserOptions, json: GLTF.IGLTF, - sourceIndex: number, - loader: ImageLoader | ImageBitmapLoader + sourceIndex?: number, + loader?: ImageLoader | ImageBitmapLoader ) => { - // eslint-disable-next-line @typescript-eslint/no-this-alias + const result = useHookstate(null) - if (sourceCache[sourceIndex] !== undefined) { - return sourceCache[sourceIndex].then((texture) => texture.clone()) - } - - const sourceDef = json.images![sourceIndex] + /** @todo caching */ + // if (sourceCache[sourceIndex] !== undefined) { + // return sourceCache[sourceIndex].then((texture) => texture.clone()) + // } - const URL = self.URL || self.webkitURL + const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null - let sourceURI = sourceDef.uri || ('' as string | Promise) + const sourceURI = useHookstate(null as null | string) let isObjectURL = false - if (sourceDef.bufferView !== undefined) { - // Load binary image data from bufferView, if provided. + const bufferViewSourceURI = GLTFLoaderFunctions.useLoadBufferView(options, json, sourceDef?.bufferView) - sourceURI = GLTFLoaderFunctions.loadBufferView(options, json, sourceDef.bufferView).then(function (bufferView) { - isObjectURL = true - const blob = new Blob([bufferView], { type: sourceDef.mimeType }) - sourceURI = URL.createObjectURL(blob) - return sourceURI - }) - } else if (sourceDef.uri === undefined) { - throw new Error('THREE.GLTFLoader: Image ' + sourceIndex + ' is missing URI and bufferView') - } + useEffect(() => { + if (!sourceDef) return - const promise = Promise.resolve(sourceURI) - .then(function (sourceURI) { - return new Promise(function (resolve, reject) { - let onLoad = resolve + if (sourceDef.uri === undefined && sourceDef.bufferView === undefined) { + console.error('THREE.GLTFLoader: Image ' + sourceIndex + ' is missing URI and bufferView') + return + } - if ((loader as ImageBitmapLoader).isImageBitmapLoader === true) { - onLoad = function (imageBitmap) { - const texture = new Texture(imageBitmap) - texture.needsUpdate = true + if (sourceDef.uri) { + const url = LoaderUtils.resolveURL(sourceDef.uri, options.path) + sourceURI.set(url) + return () => { + sourceURI.set(null) + } + } - resolve(texture) - } + if (bufferViewSourceURI) { + isObjectURL = true + const blob = new Blob([bufferViewSourceURI], { type: sourceDef.mimeType }) + const url = URL.createObjectURL(blob) + sourceURI.set(url) + return () => { + URL.revokeObjectURL(url) + sourceURI.set(null) + } + } + }, [sourceDef?.uri, bufferViewSourceURI]) + + useEffect(() => { + if (!sourceURI.value) return + console.log( + 'sourceURI', + loader, + options, + sourceURI.value, + LoaderUtils.resolveURL(sourceURI.value as string, options.path) + ) + loader!.load( + LoaderUtils.resolveURL(sourceURI.value as string, options.path), + (imageBitmap) => { + if ((loader as ImageBitmapLoader).isImageBitmapLoader === true) { + const texture = new Texture(imageBitmap) + texture.needsUpdate = true + result.set(texture) + } else { + result.set(imageBitmap) } + }, + undefined, + (e) => { + console.error(e) + console.error("THREE.GLTFLoader: Couldn't load image", sourceURI.value) + } + ) + }, [sourceURI.value]) - loader.load(LoaderUtils.resolveURL(sourceURI, options.path), onLoad, undefined, reject) - }) - }) - .then(function (texture: Texture) { - // Clean up resources and configure Texture. + useEffect(() => { + if (!result.value || !sourceURI.value || !sourceDef) return - if (isObjectURL === true) { - URL.revokeObjectURL(sourceURI as string) - } else { - texture.userData.src = sourceURI - } + const texture = result.value as Texture - texture.userData.mimeType = sourceDef.mimeType || getImageURIMimeType(sourceDef.uri) + // Clean up resources and configure Texture. - return texture - }) - .catch(function (error) { - console.error("THREE.GLTFLoader: Couldn't load texture", sourceURI) - throw error - }) + if (isObjectURL === true) { + URL.revokeObjectURL(sourceURI.value!) + } else { + texture.userData.src = sourceURI + } + + texture.userData.mimeType = sourceDef.mimeType || getImageURIMimeType(sourceDef.uri) + + // sourceCache[sourceIndex] = promise + }, [result.value]) - sourceCache[sourceIndex] = promise - return promise + return result.value as Texture | null } export const GLTFLoaderFunctions = { - loadAccessor, - loadBufferView, - loadBuffer, computeBounds, - loadMaterial, + useLoadAccessor, + useLoadBufferView, + useLoadBuffer, + useLoadMaterial, assignTexture, - loadTexture, - loadImageSource, - loadTextureImage + useLoadTexture, + useLoadImageSource, + useLoadTextureImage } diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 7eb2ebf412..680a73875a 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -429,7 +429,7 @@ export const DocumentReactor = (props: { documentID: string; parentUUID: EntityU {documentState .get(NO_PROXY) .materials?.map((material, index) => ( - { +const MaterialEntityReactor = (props: { index: number; parentUUID: EntityUUID; documentID: string }) => { const documentState = useMutableState(GLTFDocumentState)[props.documentID] const materials = documentState.materials! - const material = materials[props.index]!.get(NO_PROXY) as GLTF.IMaterial + const materialDef = materials[props.index]!.get(NO_PROXY) as GLTF.IMaterial const parentEntity = UUIDComponent.useEntityByUUID(props.parentUUID) @@ -460,24 +460,24 @@ const MaterialReactor = (props: { index: number; parentUUID: EntityUUID; documen /** Ensure all base components are added for synchronous mount */ setComponent(entity, EntityTreeComponent, { parentEntity, childIndex: props.index }) - setComponent(entity, NameComponent, material.name ?? 'Material-' + props.index) + setComponent(entity, NameComponent, materialDef.name ?? 'Material-' + props.index) entityState.set(entity) return () => { //check if entity is in some other document - if (hasComponent(entity, UUIDComponent)) { - const uuid = getComponent(entity, UUIDComponent) - const documents = getState(GLTFDocumentState) - for (const documentID in documents) { - const document = documents[documentID] - if (!document?.materials) continue - for (const material of document.materials) { - if (material.extensions?.[UUIDComponent.jsonID] === uuid) return - } - } - } - removeEntity(entity) + // if (hasComponent(entity, UUIDComponent)) { + // const uuid = getComponent(entity, UUIDComponent) + // const documents = getState(GLTFDocumentState) + // for (const documentID in documents) { + // const document = documents[documentID] + // if (!document?.materials) continue + // for (const material of document.materials) { + // if (material.extensions?.[UUIDComponent.jsonID] === uuid) return + // } + // } + // } + // removeEntity(entity) } }, []) @@ -490,20 +490,38 @@ const MaterialReactor = (props: { index: number; parentUUID: EntityUUID; documen useLayoutEffect(() => { if (!entity) return - setComponent(entity, NameComponent, material.name ?? 'Material-' + props.index) - }, [entity, material.name]) + setComponent(entity, NameComponent, materialDef.name ?? 'Material-' + props.index) + }, [entity, materialDef.name]) - useLayoutEffect(() => { - if (!entity) return + if (!entity) return null + return ( + <> + + + ) +} - GLTFLoaderFunctions.loadMaterial( - getParserOptions(entity), - documentState.get(NO_PROXY) as GLTF.IGLTF, - props.index - ).then((material) => { - setComponent(entity, MaterialStateComponent, { material }) - }) - }, [entity, material]) +const MaterialStateReactor = (props: { index: number; parentUUID: EntityUUID; documentID: string; entity: Entity }) => { + const documentState = useMutableState(GLTFDocumentState)[props.documentID] + const entity = props.entity + + const material = GLTFLoaderFunctions.useLoadMaterial( + getParserOptions(entity), + documentState.get(NO_PROXY) as GLTF.IGLTF, + props.index + ) + + useLayoutEffect(() => { + if (!entity || !material) return + const uuid = getComponent(entity, UUIDComponent) + material.uuid = uuid + setComponent(entity, MaterialStateComponent, { material }) + }, [material]) return null } @@ -836,23 +854,23 @@ const PrimitiveAttributeReactor = (props: { const primitive = mesh.primitives[props.primitiveIndex] - const attribute = primitive.attributes[props.attribute] + const threeAttributeName = ATTRIBUTES[props.attribute] || props.attribute.toLowerCase() + const attributeAlreadyLoaded = threeAttributeName in props.geometry.attributes - useEffect(() => { - const threeAttributeName = ATTRIBUTES[props.attribute] || props.attribute.toLowerCase() + // Skip attributes already provided by e.g. Draco extension. + const attribute = attributeAlreadyLoaded ? undefined : primitive.attributes[props.attribute] - // Skip attributes already provided by e.g. Draco extension. - if (threeAttributeName in props.geometry.attributes) return + const accessor = GLTFLoaderFunctions.useLoadAccessor( + getParserOptions(props.entity), + documentState.get(NO_PROXY) as GLTF.IGLTF, + attribute + ) - // accessor - GLTFLoaderFunctions.loadAccessor( - getParserOptions(props.entity), - documentState.get(NO_PROXY) as GLTF.IGLTF, - attribute - ).then((accessor) => { - props.geometry.setAttribute(threeAttributeName, accessor) - }) - }, [attribute, props.geometry]) + useEffect(() => { + if (!accessor) return + + props.geometry.setAttribute(threeAttributeName, accessor) + }, [accessor, props.geometry]) return null } @@ -873,16 +891,16 @@ const PrimitiveIndicesAttributeReactor = (props: { const primitive = mesh.primitives[props.primitiveIndex] + const accessor = GLTFLoaderFunctions.useLoadAccessor( + getParserOptions(props.entity), + documentState.get(NO_PROXY) as GLTF.IGLTF, + primitive.indices! + ) + useEffect(() => { - // accessor - GLTFLoaderFunctions.loadAccessor( - getParserOptions(props.entity), - documentState.get(NO_PROXY) as GLTF.IGLTF, - primitive.indices! - ).then((accessor) => { - props.geometry.setIndex(accessor) - }) - }, [primitive.indices, props.geometry]) + if (!accessor) return + props.geometry.setIndex(accessor) + }, [accessor, props.geometry]) return null } From fafc991e767accbe4478c9eb6c9ed118d38818c9 Mon Sep 17 00:00:00 2001 From: HexaField Date: Thu, 1 Aug 2024 14:12:52 +1000 Subject: [PATCH 09/47] fix texture loading --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 26 +++++++------------ .../systems/MaterialLibrarySystem.tsx | 24 ++++++----------- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 70dac3a65c..618814eaa4 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -399,7 +399,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI // Specification: // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#metallic-roughness-material - const map = GLTFLoaderFunctions.assignTexture(options, json, materialDef.pbrMetallicRoughness?.baseColorTexture) + const map = GLTFLoaderFunctions.useAssignTexture(options, json, materialDef.pbrMetallicRoughness?.baseColorTexture) useEffect(() => { if (!map) return @@ -435,7 +435,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI }) }, [materialDef.pbrMetallicRoughness?.roughnessFactor]) - const metalnessMap = GLTFLoaderFunctions.assignTexture( + const metalnessMap = GLTFLoaderFunctions.useAssignTexture( options, json, materialType.value === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture @@ -446,7 +446,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI result.value?.setValues({ metalnessMap }) }, [metalnessMap]) - const roughnessMap = GLTFLoaderFunctions.assignTexture( + const roughnessMap = GLTFLoaderFunctions.useAssignTexture( options, json, materialType.value === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture @@ -479,7 +479,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI } }, [materialDef.alphaCutoff]) - const normalMap = GLTFLoaderFunctions.assignTexture( + const normalMap = GLTFLoaderFunctions.useAssignTexture( options, json, materialType.value === 'basic' ? undefined : materialDef.normalTexture @@ -500,7 +500,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI // } // }, []) - const aoMap = GLTFLoaderFunctions.assignTexture( + const aoMap = GLTFLoaderFunctions.useAssignTexture( options, json, materialType.value === 'basic' ? undefined : materialDef.occlusionTexture @@ -524,7 +524,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI }) }, [materialDef.emissiveFactor]) - const emissiveMap = GLTFLoaderFunctions.assignTexture( + const emissiveMap = GLTFLoaderFunctions.useAssignTexture( options, json, materialType.value === 'basic' ? undefined : materialDef.emissiveTexture @@ -546,7 +546,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI * @param {Object} mapDef * @return {Promise} */ -const assignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, mapDef?: GLTF.ITextureInfo) => { +const useAssignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, mapDef?: GLTF.ITextureInfo) => { const result = useHookstate(null) const texture = GLTFLoaderFunctions.useLoadTexture(options, json, mapDef?.index) @@ -565,6 +565,8 @@ const assignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, mapDef?: GL result.set(textureClone) } + result.set(texture) + if (GLTFExtensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM]) { const transform = mapDef.extensions !== undefined ? mapDef.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM] : undefined @@ -629,7 +631,6 @@ const useLoadTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, textureInd } const texture = GLTFLoaderFunctions.useLoadTextureImage(options, json, textureIndex, sourceIndex, loader) - useEffect(() => { result.set(texture) }, [texture]) @@ -745,13 +746,6 @@ const useLoadImageSource = ( useEffect(() => { if (!sourceURI.value) return - console.log( - 'sourceURI', - loader, - options, - sourceURI.value, - LoaderUtils.resolveURL(sourceURI.value as string, options.path) - ) loader!.load( LoaderUtils.resolveURL(sourceURI.value as string, options.path), (imageBitmap) => { @@ -798,7 +792,7 @@ export const GLTFLoaderFunctions = { useLoadBufferView, useLoadBuffer, useLoadMaterial, - assignTexture, + useAssignTexture, useLoadTexture, useLoadImageSource, useLoadTextureImage diff --git a/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx b/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx index fb427ab80f..3fbda5837a 100644 --- a/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx +++ b/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx @@ -35,8 +35,7 @@ import { setComponent, UndefinedEntity, useComponent, - useEntityContext, - useOptionalComponent + useEntityContext } from '@etherealengine/ecs' import { defineSystem } from '@etherealengine/ecs/src/SystemFunctions' import { @@ -72,16 +71,15 @@ const reactor = (): ReactElement => { return ( <> - + - ) } const MeshReactor = () => { const entity = useEntityContext() - const materialComponent = useOptionalComponent(entity, MaterialInstanceComponent) + const materialComponent = useComponent(entity, MaterialInstanceComponent) const meshComponent = useComponent(entity, MeshComponent) const createAndSourceMaterial = (material: Material) => { @@ -91,11 +89,15 @@ const MeshReactor = () => { } useEffect(() => { - if (materialComponent) return const material = meshComponent.material.value as Material if (!isArray(material)) createAndSourceMaterial(material) else for (const mat of material) createAndSourceMaterial(mat) }, []) + + useEffect(() => { + const uuid = materialComponent.uuid.value + if (uuid) setMeshMaterial(entity, uuid as EntityUUID[]) + }, [materialComponent.uuid]) return null } @@ -125,16 +127,6 @@ const MaterialEntityReactor = () => { return null } -const MaterialInstanceReactor = () => { - const entity = useEntityContext() - const materialComponent = useComponent(entity, MaterialInstanceComponent) - const uuid = materialComponent.uuid - useEffect(() => { - if (uuid.value) setMeshMaterial(entity, uuid.value as EntityUUID[]) - }, [materialComponent.uuid]) - return null -} - export const MaterialLibrarySystem = defineSystem({ uuid: 'ee.engine.scene.MaterialLibrarySystem', insert: { after: PresentationSystemGroup }, From 9121c488e1651dd75a01a5c137bef2da9ab9a572 Mon Sep 17 00:00:00 2001 From: HexaField Date: Thu, 1 Aug 2024 15:48:49 +1000 Subject: [PATCH 10/47] move and simplify reactors for material components --- .../systems/MaterialLibrarySystem.tsx | 87 ++----------------- ...rialComponent.ts => MaterialComponent.tsx} | 60 ++++++++++++- 2 files changed, 63 insertions(+), 84 deletions(-) rename packages/spatial/src/renderer/materials/{MaterialComponent.ts => MaterialComponent.tsx} (75%) diff --git a/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx b/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx index 3fbda5837a..9fa9c29dd7 100644 --- a/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx +++ b/packages/engine/src/scene/materials/systems/MaterialLibrarySystem.tsx @@ -23,20 +23,9 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import React, { ReactElement, useEffect } from 'react' +import { useEffect } from 'react' -import { - EntityUUID, - getComponent, - getOptionalComponent, - PresentationSystemGroup, - QueryReactor, - removeEntity, - setComponent, - UndefinedEntity, - useComponent, - useEntityContext -} from '@etherealengine/ecs' +import { PresentationSystemGroup, UndefinedEntity } from '@etherealengine/ecs' import { defineSystem } from '@etherealengine/ecs/src/SystemFunctions' import { MaterialPrototypeDefinition, @@ -44,22 +33,13 @@ import { } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { createAndAssignMaterial, - createMaterialPrototype, - materialPrototypeMatches, - setMeshMaterial, - updateMaterialPrototype + createMaterialPrototype } from '@etherealengine/spatial/src/renderer/materials/materialFunctions' -import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' -import { - MaterialInstanceComponent, - MaterialStateComponent -} from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' -import { isArray } from 'lodash' -import { Material, MeshBasicMaterial } from 'three' -import { SourceComponent } from '../../components/SourceComponent' +import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { MeshBasicMaterial } from 'three' -const reactor = (): ReactElement => { +const reactor = () => { useEffect(() => { MaterialPrototypeDefinitions.map((prototype: MaterialPrototypeDefinition, uuid) => createMaterialPrototype(prototype) @@ -69,61 +49,6 @@ const reactor = (): ReactElement => { createAndAssignMaterial(UndefinedEntity, fallbackMaterial) }, []) - return ( - <> - - - - ) -} - -const MeshReactor = () => { - const entity = useEntityContext() - const materialComponent = useComponent(entity, MaterialInstanceComponent) - const meshComponent = useComponent(entity, MeshComponent) - - const createAndSourceMaterial = (material: Material) => { - const materialEntity = createAndAssignMaterial(entity, material) - const source = getOptionalComponent(entity, SourceComponent) - if (source) setComponent(materialEntity, SourceComponent, source) - } - - useEffect(() => { - const material = meshComponent.material.value as Material - if (!isArray(material)) createAndSourceMaterial(material) - else for (const mat of material) createAndSourceMaterial(mat) - }, []) - - useEffect(() => { - const uuid = materialComponent.uuid.value - if (uuid) setMeshMaterial(entity, uuid as EntityUUID[]) - }, [materialComponent.uuid]) - return null -} - -const MaterialEntityReactor = () => { - const entity = useEntityContext() - const materialComponent = useComponent(entity, MaterialStateComponent) - useEffect(() => { - if (!materialComponent.instances.value!) return - for (const sourceEntity of materialComponent.instances.value) { - const sourceComponent = getComponent(sourceEntity, SourceComponent) - if (!SourceComponent.entitiesBySource[sourceComponent]) return - for (const entity of SourceComponent.entitiesBySource[getComponent(sourceEntity, SourceComponent)]) { - const uuid = getOptionalComponent(entity, MaterialInstanceComponent)?.uuid as EntityUUID[] | undefined - if (uuid) setMeshMaterial(entity, uuid) - } - } - }, [materialComponent.material]) - - useEffect(() => { - if (materialComponent.prototypeEntity.value && !materialPrototypeMatches(entity)) updateMaterialPrototype(entity) - }, [materialComponent.prototypeEntity]) - - useEffect(() => { - if (materialComponent.instances.value?.length === 0) removeEntity(entity) - }, [materialComponent.instances]) - return null } diff --git a/packages/spatial/src/renderer/materials/MaterialComponent.ts b/packages/spatial/src/renderer/materials/MaterialComponent.tsx similarity index 75% rename from packages/spatial/src/renderer/materials/MaterialComponent.ts rename to packages/spatial/src/renderer/materials/MaterialComponent.tsx index ada1a7f4cb..b625f2a73f 100644 --- a/packages/spatial/src/renderer/materials/MaterialComponent.ts +++ b/packages/spatial/src/renderer/materials/MaterialComponent.tsx @@ -23,7 +23,7 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { Material, Shader, WebGLRenderer } from 'three' +import { Material, Mesh, Shader, WebGLRenderer } from 'three' import { Component, @@ -31,15 +31,19 @@ import { defineComponent, defineQuery, getComponent, - getMutableComponent + getMutableComponent, + useComponent, + useEntityContext } from '@etherealengine/ecs' import { Entity, EntityUUID, UndefinedEntity } from '@etherealengine/ecs/src/Entity' import { PluginType } from '@etherealengine/spatial/src/common/functions/OnBeforeCompilePlugin' +import React, { useEffect } from 'react' import { v4 as uuidv4 } from 'uuid' +import { MeshComponent } from '../components/MeshComponent' import { NoiseOffsetPlugin } from './constants/plugins/NoiseOffsetPlugin' import { TransparencyDitheringPlugin } from './constants/plugins/TransparencyDitheringComponent' -import { setMeshMaterial } from './materialFunctions' +import { materialPrototypeMatches, setMeshMaterial, updateMaterialPrototype } from './materialFunctions' import MeshBasicMaterial from './prototypes/MeshBasicMaterial.mat' import MeshLambertMaterial from './prototypes/MeshLambertMaterial.mat' import MeshMatcapMaterial from './prototypes/MeshMatcapMaterial.mat' @@ -115,6 +119,17 @@ export const MaterialStateComponent = defineComponent({ for (const instanceEntity of materialComponent.instances) { setMeshMaterial(instanceEntity, getComponent(instanceEntity, MaterialInstanceComponent).uuid) } + }, + + reactor: () => { + const entity = useEntityContext() + const materialComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + if (materialComponent.prototypeEntity.value && !materialPrototypeMatches(entity)) updateMaterialPrototype(entity) + }, [materialComponent.prototypeEntity]) + + return null } }) @@ -136,9 +151,48 @@ export const MaterialInstanceComponent = defineComponent({ if (materialComponent.instances.value) materialComponent.instances.set(materialComponent.instances.value.filter((instance) => instance !== entity)) } + }, + reactor: () => { + const entity = useEntityContext() + const materialComponent = useComponent(entity, MaterialInstanceComponent) + const meshComponent = useComponent(entity, MeshComponent) + + if (Array.isArray(meshComponent.material.value)) + return ( + <> + {materialComponent.uuid.value.map((uuid, index) => ( + + ))} + + ) + + return ( + + ) } }) +const MaterialInstanceSubReactor = (props: { uuid: EntityUUID; entity: Entity; index: number }) => { + const { uuid, entity, index } = props + const materialStateEntity = UUIDComponent.useEntityByUUID(uuid) + const materialStateComponent = useComponent(materialStateEntity, MaterialStateComponent) + const meshComponent = useComponent(entity, MeshComponent) + + useEffect(() => { + const mesh = meshComponent.value as Mesh + const material = materialStateComponent.material.value as Material + if (Array.isArray(mesh.material)) mesh.material[index] = material + mesh.material = material + }, [materialStateComponent.material]) + + return null +} + export const MaterialPrototypeComponent = defineComponent({ name: 'MaterialPrototypeComponent', onInit: (entity) => { From 0990662f48fdef34520364b271b99781ad6f5808 Mon Sep 17 00:00:00 2001 From: HexaField Date: Fri, 2 Aug 2024 10:12:06 +1000 Subject: [PATCH 11/47] fixes --- packages/engine/src/gltf/GLTFState.tsx | 26 +++++++++---------- .../renderer/materials/MaterialComponent.tsx | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 680a73875a..570e8a4074 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -466,18 +466,18 @@ const MaterialEntityReactor = (props: { index: number; parentUUID: EntityUUID; d return () => { //check if entity is in some other document - // if (hasComponent(entity, UUIDComponent)) { - // const uuid = getComponent(entity, UUIDComponent) - // const documents = getState(GLTFDocumentState) - // for (const documentID in documents) { - // const document = documents[documentID] - // if (!document?.materials) continue - // for (const material of document.materials) { - // if (material.extensions?.[UUIDComponent.jsonID] === uuid) return - // } - // } - // } - // removeEntity(entity) + if (hasComponent(entity, UUIDComponent)) { + const uuid = getComponent(entity, UUIDComponent) + const documents = getState(GLTFDocumentState) + for (const documentID in documents) { + const document = documents[documentID] + if (!document?.materials) continue + for (const material of document.materials) { + if (material.extensions?.[UUIDComponent.jsonID] === uuid) return + } + } + } + removeEntity(entity) } }, []) @@ -541,7 +541,7 @@ const ParentNodeReactor = (props: { const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: EntityUUID; documentID: string }) => { const documentState = useMutableState(GLTFDocumentState)[props.documentID] - const nodes = documentState.nodes! // as State + const nodes = documentState.nodes! const node = nodes[props.nodeIndex]! as State diff --git a/packages/spatial/src/renderer/materials/MaterialComponent.tsx b/packages/spatial/src/renderer/materials/MaterialComponent.tsx index b625f2a73f..0366bf7699 100644 --- a/packages/spatial/src/renderer/materials/MaterialComponent.tsx +++ b/packages/spatial/src/renderer/materials/MaterialComponent.tsx @@ -187,7 +187,7 @@ const MaterialInstanceSubReactor = (props: { uuid: EntityUUID; entity: Entity; i const mesh = meshComponent.value as Mesh const material = materialStateComponent.material.value as Material if (Array.isArray(mesh.material)) mesh.material[index] = material - mesh.material = material + else mesh.material = material }, [materialStateComponent.material]) return null From dcfe4ce90250654d17f704cf23a00ab5592e86d4 Mon Sep 17 00:00:00 2001 From: HexaField Date: Fri, 2 Aug 2024 19:02:44 +1000 Subject: [PATCH 12/47] needs update upon value change --- packages/engine/src/gltf/GLTFLoaderFunctions.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 618814eaa4..84f70c8fc9 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -371,6 +371,8 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI result.set(material) }, [materialType]) + const material = result.get(NO_PROXY) as MeshStandardMaterial | MeshBasicMaterial + const materialDef = json.materials![materialIndex] const materialExtensions = materialDef.extensions || {} @@ -405,6 +407,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI if (!map) return map.colorSpace = SRGBColorSpace result.value?.setValues({ map }) + material.needsUpdate = true }, [map]) useEffect(() => { @@ -414,6 +417,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI color: new Color().setRGB(array[0], array[1], array[2], LinearSRGBColorSpace), opacity: array[3] }) + material.needsUpdate = true } }, [materialDef.pbrMetallicRoughness?.baseColorFactor]) @@ -424,6 +428,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI ? materialDef.pbrMetallicRoughness.metallicFactor : 1.0 }) + material.needsUpdate = true }, [materialDef.pbrMetallicRoughness?.metallicFactor]) useEffect(() => { @@ -433,6 +438,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI ? materialDef.pbrMetallicRoughness.roughnessFactor : 1.0 }) + material.needsUpdate = true }, [materialDef.pbrMetallicRoughness?.roughnessFactor]) const metalnessMap = GLTFLoaderFunctions.useAssignTexture( @@ -444,6 +450,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI useEffect(() => { if (!metalnessMap) return result.value?.setValues({ metalnessMap }) + material.needsUpdate = true }, [metalnessMap]) const roughnessMap = GLTFLoaderFunctions.useAssignTexture( @@ -455,10 +462,12 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI useEffect(() => { if (!roughnessMap) return result.value?.setValues({ roughnessMap }) + material.needsUpdate = true }, [roughnessMap]) useEffect(() => { result.value?.setValues({ side: materialDef.doubleSided === true ? DoubleSide : FrontSide }) + material.needsUpdate = true }, [materialDef.doubleSided]) useEffect(() => { @@ -469,6 +478,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI if (alphaMode !== ALPHA_MODES.OPAQUE) { result.value?.setValues({ depthWrite: false }) } + material.needsUpdate = true }, [materialDef.alphaMode]) useEffect(() => { @@ -477,6 +487,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI } else { result.value?.setValues({ alphaTest: 0 }) } + material.needsUpdate = true }, [materialDef.alphaCutoff]) const normalMap = GLTFLoaderFunctions.useAssignTexture( @@ -488,6 +499,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI useEffect(() => { if (!normalMap) return result.value?.setValues({ normalMap }) + material.needsUpdate = true }, [normalMap]) // useEffect(() => { @@ -509,10 +521,12 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI useEffect(() => { if (!aoMap) return result.value?.setValues({ aoMap }) + material.needsUpdate = true }, [aoMap]) useEffect(() => { result.value?.setValues({ aoMapIntensity: materialDef.occlusionTexture?.strength ?? 1.0 }) + material.needsUpdate = true }, [materialDef.occlusionTexture?.strength]) useEffect(() => { @@ -522,6 +536,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI result.value?.setValues({ emissive: new Color().setRGB(emissiveFactor[0], emissiveFactor[1], emissiveFactor[2], LinearSRGBColorSpace) }) + material.needsUpdate = true }, [materialDef.emissiveFactor]) const emissiveMap = GLTFLoaderFunctions.useAssignTexture( @@ -534,6 +549,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI if (!emissiveMap) return emissiveMap.colorSpace = SRGBColorSpace result.value?.setValues({ emissiveMap }) + material.needsUpdate = true }, [emissiveMap]) return result.value as MeshStandardMaterial | null From 6f8439f71789f2452f7f59ae539956522408cb1b Mon Sep 17 00:00:00 2001 From: HexaField Date: Fri, 2 Aug 2024 19:03:34 +1000 Subject: [PATCH 13/47] needs update upon value change --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 84f70c8fc9..8dfb366628 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -407,7 +407,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI if (!map) return map.colorSpace = SRGBColorSpace result.value?.setValues({ map }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [map]) useEffect(() => { @@ -417,7 +417,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI color: new Color().setRGB(array[0], array[1], array[2], LinearSRGBColorSpace), opacity: array[3] }) - material.needsUpdate = true + if (material) material.needsUpdate = true } }, [materialDef.pbrMetallicRoughness?.baseColorFactor]) @@ -428,7 +428,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI ? materialDef.pbrMetallicRoughness.metallicFactor : 1.0 }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [materialDef.pbrMetallicRoughness?.metallicFactor]) useEffect(() => { @@ -438,7 +438,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI ? materialDef.pbrMetallicRoughness.roughnessFactor : 1.0 }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [materialDef.pbrMetallicRoughness?.roughnessFactor]) const metalnessMap = GLTFLoaderFunctions.useAssignTexture( @@ -450,7 +450,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI useEffect(() => { if (!metalnessMap) return result.value?.setValues({ metalnessMap }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [metalnessMap]) const roughnessMap = GLTFLoaderFunctions.useAssignTexture( @@ -462,12 +462,12 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI useEffect(() => { if (!roughnessMap) return result.value?.setValues({ roughnessMap }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [roughnessMap]) useEffect(() => { result.value?.setValues({ side: materialDef.doubleSided === true ? DoubleSide : FrontSide }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [materialDef.doubleSided]) useEffect(() => { @@ -478,7 +478,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI if (alphaMode !== ALPHA_MODES.OPAQUE) { result.value?.setValues({ depthWrite: false }) } - material.needsUpdate = true + if (material) material.needsUpdate = true }, [materialDef.alphaMode]) useEffect(() => { @@ -487,7 +487,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI } else { result.value?.setValues({ alphaTest: 0 }) } - material.needsUpdate = true + if (material) material.needsUpdate = true }, [materialDef.alphaCutoff]) const normalMap = GLTFLoaderFunctions.useAssignTexture( @@ -499,7 +499,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI useEffect(() => { if (!normalMap) return result.value?.setValues({ normalMap }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [normalMap]) // useEffect(() => { @@ -521,12 +521,12 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI useEffect(() => { if (!aoMap) return result.value?.setValues({ aoMap }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [aoMap]) useEffect(() => { result.value?.setValues({ aoMapIntensity: materialDef.occlusionTexture?.strength ?? 1.0 }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [materialDef.occlusionTexture?.strength]) useEffect(() => { @@ -536,7 +536,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI result.value?.setValues({ emissive: new Color().setRGB(emissiveFactor[0], emissiveFactor[1], emissiveFactor[2], LinearSRGBColorSpace) }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [materialDef.emissiveFactor]) const emissiveMap = GLTFLoaderFunctions.useAssignTexture( @@ -549,7 +549,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI if (!emissiveMap) return emissiveMap.colorSpace = SRGBColorSpace result.value?.setValues({ emissiveMap }) - material.needsUpdate = true + if (material) material.needsUpdate = true }, [emissiveMap]) return result.value as MeshStandardMaterial | null From 0b4f3fafca22cbd7e4c9b0f144b8c98222de89bf Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 13:01:49 +1000 Subject: [PATCH 14/47] support material extensions --- .../src/assets/loaders/gltf/GLTFParser.ts | 1 + .../engine/src/gltf/GLTFLoaderFunctions.ts | 119 ++- packages/engine/src/gltf/GLTFState.tsx | 70 +- .../src/gltf/MaterialDefinitionComponent.tsx | 811 ++++++++++++++++++ 4 files changed, 916 insertions(+), 85 deletions(-) create mode 100644 packages/engine/src/gltf/MaterialDefinitionComponent.tsx diff --git a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts index aa20ed3078..29bbf7832e 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts @@ -125,6 +125,7 @@ declare module '@gltf-transform/core/dist/types/gltf.d.ts' { export type GLTFParserOptions = { body: null | ArrayBuffer + document: GLTF.IGLTF crossOrigin: 'anonymous' | string ktx2Loader: KTX2Loader manager: LoadingManager diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 8dfb366628..3aeeddd235 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -23,6 +23,7 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ +import { ComponentType } from '@etherealengine/ecs' import { NO_PROXY, useHookstate } from '@etherealengine/hyperflux' import { GLTF } from '@gltf-transform/core' import { useEffect } from 'react' @@ -42,6 +43,7 @@ import { LinearSRGBColorSpace, LoaderUtils, MeshBasicMaterial, + MeshPhysicalMaterial, MeshStandardMaterial, RepeatWrapping, SRGBColorSpace, @@ -59,37 +61,30 @@ import { WEBGL_WRAPPINGS } from '../assets/loaders/gltf/GLTFConstants' import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' -import { - addUnknownExtensionsToUserData, - assignExtrasToUserData, - getNormalizedComponentScale -} from '../assets/loaders/gltf/GLTFLoaderFunctions' +import { assignExtrasToUserData, getNormalizedComponentScale } from '../assets/loaders/gltf/GLTFLoaderFunctions' import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' import { GLTFExtensions } from './GLTFExtensions' +import { MaterialDefinitionComponent } from './MaterialDefinitionComponent' // todo make this a state const cache = new GLTFRegistry() -const useLoadAccessor = (options: GLTFParserOptions, json: GLTF.IGLTF, accessorIndex?: number) => { +const useLoadAccessor = (options: GLTFParserOptions, accessorIndex?: number) => { + const json = options.document const result = useHookstate(null) const accessorDef = typeof accessorIndex === 'number' ? json.accessors![accessorIndex] : null - const bufferView = GLTFLoaderFunctions.useLoadBufferView(options, json, accessorDef?.bufferView) + const bufferView = GLTFLoaderFunctions.useLoadBufferView(options, accessorDef?.bufferView) const sparseBufferViewIndices = GLTFLoaderFunctions.useLoadBufferView( options, - json, accessorDef?.sparse?.indices?.bufferView ) - const sparseBufferViewValues = GLTFLoaderFunctions.useLoadBufferView( - options, - json, - accessorDef?.sparse?.values?.bufferView - ) + const sparseBufferViewValues = GLTFLoaderFunctions.useLoadBufferView(options, accessorDef?.sparse?.values?.bufferView) useEffect(() => { - if (!accessorDef) return + if (!accessorDef || !bufferView) return if (accessorDef.bufferView === undefined && accessorDef.sparse === undefined) { const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] @@ -200,11 +195,12 @@ const useLoadAccessor = (options: GLTFParserOptions, json: GLTF.IGLTF, accessorI return result.value } -const useLoadBufferView = (options: GLTFParserOptions, json: GLTF.IGLTF, bufferViewIndex?: number) => { +const useLoadBufferView = (options: GLTFParserOptions, bufferViewIndex?: number) => { + const json = options.document const result = useHookstate(null) const bufferViewDef = typeof bufferViewIndex === 'number' ? json.bufferViews![bufferViewIndex] : null - const buffer = GLTFLoaderFunctions.useLoadBuffer(options, json, bufferViewDef?.buffer) + const buffer = GLTFLoaderFunctions.useLoadBuffer(options, bufferViewDef?.buffer) useEffect(() => { if (!bufferViewDef || !buffer) return @@ -218,7 +214,8 @@ const useLoadBufferView = (options: GLTFParserOptions, json: GLTF.IGLTF, bufferV return result.value } -const useLoadBuffer = (options: GLTFParserOptions, json: GLTF.IGLTF, bufferIndex?: number) => { +const useLoadBuffer = (options: GLTFParserOptions, bufferIndex?: number) => { + const json = options.document const result = useHookstate(null) const bufferDef = typeof bufferIndex === 'number' ? json.buffers![bufferIndex] : null @@ -345,18 +342,25 @@ export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primit geometry.boundingSphere = sphere } +const Prototypes = { + basic: MeshBasicMaterial, + standard: MeshStandardMaterial, + physical: MeshPhysicalMaterial +} + /** * Specification: https://github.com/KhronosGroup/glTF/blob/master/specification/2.0/README.md#materials * @param {number} materialIndex * @return {Promise} */ -const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialIndex: number) => { - const materialType = useHookstate<'standard' | 'basic'>('standard') - +const useLoadMaterial = ( + options: GLTFParserOptions, + materialDef: ComponentType +) => { const result = useHookstate(null as null | MeshStandardMaterial | MeshBasicMaterial) useEffect(() => { - const materialTypeValue = materialType.get(NO_PROXY) === 'standard' ? MeshStandardMaterial : MeshBasicMaterial + const materialTypeValue = Prototypes[materialDef.type] const material = new materialTypeValue() result.set(material) @@ -366,15 +370,13 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI /** @todo */ // parser.associations.set(material, { materials: materialIndex }) - if (materialDef.extensions) addUnknownExtensionsToUserData(GLTFExtensions, material, materialDef) + // if (materialDef.extensions) addUnknownExtensionsToUserData(GLTFExtensions, material, materialDef) result.set(material) - }, [materialType]) + }, [materialDef.type]) const material = result.get(NO_PROXY) as MeshStandardMaterial | MeshBasicMaterial - const materialDef = json.materials![materialIndex] - const materialExtensions = materialDef.extensions || {} /** @todo expose 'getMaterialType' API */ @@ -396,19 +398,19 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI // if (!materialExtensions[EXTENSIONS.EE_MATERIAL] && materialExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT]) { // const kmuExtension = GLTFExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT] // materialType = kmuExtension.getMaterialType() - // pending.push(kmuExtension.extendParams(options, json, materialParams, materialDef)) + // pending.push(kmuExtension.extendParams(options, materialParams, materialDef)) // } else { // Specification: // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#metallic-roughness-material - const map = GLTFLoaderFunctions.useAssignTexture(options, json, materialDef.pbrMetallicRoughness?.baseColorTexture) + const map = GLTFLoaderFunctions.useAssignTexture(options, materialDef.pbrMetallicRoughness?.baseColorTexture) useEffect(() => { if (!map) return map.colorSpace = SRGBColorSpace result.value?.setValues({ map }) if (material) material.needsUpdate = true - }, [map]) + }, [material, map]) useEffect(() => { if (Array.isArray(materialDef.pbrMetallicRoughness?.baseColorFactor)) { @@ -419,7 +421,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI }) if (material) material.needsUpdate = true } - }, [materialDef.pbrMetallicRoughness?.baseColorFactor]) + }, [material, materialDef.pbrMetallicRoughness?.baseColorFactor]) useEffect(() => { result.value?.setValues({ @@ -429,7 +431,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI : 1.0 }) if (material) material.needsUpdate = true - }, [materialDef.pbrMetallicRoughness?.metallicFactor]) + }, [material, materialDef.pbrMetallicRoughness?.metallicFactor]) useEffect(() => { result.value?.setValues({ @@ -439,36 +441,34 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI : 1.0 }) if (material) material.needsUpdate = true - }, [materialDef.pbrMetallicRoughness?.roughnessFactor]) + }, [material, materialDef.pbrMetallicRoughness?.roughnessFactor]) const metalnessMap = GLTFLoaderFunctions.useAssignTexture( options, - json, - materialType.value === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture + materialDef.type === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture ) useEffect(() => { if (!metalnessMap) return result.value?.setValues({ metalnessMap }) if (material) material.needsUpdate = true - }, [metalnessMap]) + }, [material, metalnessMap]) const roughnessMap = GLTFLoaderFunctions.useAssignTexture( options, - json, - materialType.value === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture + materialDef.type === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture ) useEffect(() => { if (!roughnessMap) return result.value?.setValues({ roughnessMap }) if (material) material.needsUpdate = true - }, [roughnessMap]) + }, [material, roughnessMap]) useEffect(() => { result.value?.setValues({ side: materialDef.doubleSided === true ? DoubleSide : FrontSide }) if (material) material.needsUpdate = true - }, [materialDef.doubleSided]) + }, [material, materialDef.doubleSided]) useEffect(() => { const alphaMode = materialDef.alphaMode || ALPHA_MODES.OPAQUE @@ -479,7 +479,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI result.value?.setValues({ depthWrite: false }) } if (material) material.needsUpdate = true - }, [materialDef.alphaMode]) + }, [material, materialDef.alphaMode]) useEffect(() => { if (materialDef.alphaMode === ALPHA_MODES.MASK) { @@ -488,19 +488,18 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI result.value?.setValues({ alphaTest: 0 }) } if (material) material.needsUpdate = true - }, [materialDef.alphaCutoff]) + }, [material, materialDef.alphaCutoff]) const normalMap = GLTFLoaderFunctions.useAssignTexture( options, - json, - materialType.value === 'basic' ? undefined : materialDef.normalTexture + materialDef.type === 'basic' ? undefined : materialDef.normalTexture ) useEffect(() => { if (!normalMap) return result.value?.setValues({ normalMap }) if (material) material.needsUpdate = true - }, [normalMap]) + }, [material, normalMap]) // useEffect(() => { // materialParams.normalScale = new Vector2(1, 1) @@ -510,24 +509,23 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI // materialParams.normalScale.set(scale, scale) // } - // }, []) + // }, [material, ]) const aoMap = GLTFLoaderFunctions.useAssignTexture( options, - json, - materialType.value === 'basic' ? undefined : materialDef.occlusionTexture + materialDef.type === 'basic' ? undefined : materialDef.occlusionTexture ) useEffect(() => { if (!aoMap) return result.value?.setValues({ aoMap }) if (material) material.needsUpdate = true - }, [aoMap]) + }, [material, aoMap]) useEffect(() => { result.value?.setValues({ aoMapIntensity: materialDef.occlusionTexture?.strength ?? 1.0 }) if (material) material.needsUpdate = true - }, [materialDef.occlusionTexture?.strength]) + }, [material, materialDef.occlusionTexture?.strength]) useEffect(() => { const emissiveFactor = materialDef.emissiveFactor @@ -537,12 +535,11 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI emissive: new Color().setRGB(emissiveFactor[0], emissiveFactor[1], emissiveFactor[2], LinearSRGBColorSpace) }) if (material) material.needsUpdate = true - }, [materialDef.emissiveFactor]) + }, [material, materialDef.emissiveFactor]) const emissiveMap = GLTFLoaderFunctions.useAssignTexture( options, - json, - materialType.value === 'basic' ? undefined : materialDef.emissiveTexture + materialDef.type === 'basic' ? undefined : materialDef.emissiveTexture ) useEffect(() => { @@ -550,7 +547,7 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI emissiveMap.colorSpace = SRGBColorSpace result.value?.setValues({ emissiveMap }) if (material) material.needsUpdate = true - }, [emissiveMap]) + }, [material, emissiveMap]) return result.value as MeshStandardMaterial | null } @@ -562,10 +559,11 @@ const useLoadMaterial = (options: GLTFParserOptions, json: GLTF.IGLTF, materialI * @param {Object} mapDef * @return {Promise} */ -const useAssignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, mapDef?: GLTF.ITextureInfo) => { +const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo) => { + const json = options.document const result = useHookstate(null) - const texture = GLTFLoaderFunctions.useLoadTexture(options, json, mapDef?.index) + const texture = GLTFLoaderFunctions.useLoadTexture(options, mapDef?.index) useEffect(() => { if (!texture) { @@ -604,7 +602,8 @@ const useAssignTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, mapDef?: * @param {number} textureIndex * @return {Promise} */ -const useLoadTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, textureIndex?: number) => { +const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { + const json = options.document const result = useHookstate(null) const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null @@ -646,7 +645,7 @@ const useLoadTexture = (options: GLTFParserOptions, json: GLTF.IGLTF, textureInd if (handler !== null) loader = handler } - const texture = GLTFLoaderFunctions.useLoadTextureImage(options, json, textureIndex, sourceIndex, loader) + const texture = GLTFLoaderFunctions.useLoadTextureImage(options, textureIndex, sourceIndex, loader) useEffect(() => { result.set(texture) }, [texture]) @@ -658,11 +657,11 @@ const textureCache = {} as any // todo const useLoadTextureImage = ( options: GLTFParserOptions, - json: GLTF.IGLTF, textureIndex?: number, sourceIndex?: number, loader?: ImageLoader | ImageBitmapLoader ) => { + const json = options.document const result = useHookstate(null) const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null @@ -676,7 +675,7 @@ const useLoadTextureImage = ( // return textureCache[cacheKey] // } - const texture = GLTFLoaderFunctions.useLoadImageSource(options, json, sourceIndex, loader) + const texture = GLTFLoaderFunctions.useLoadImageSource(options, sourceIndex, loader) useEffect(() => { if (!texture || !sourceDef || !textureDef) return @@ -714,10 +713,10 @@ const URL = self.URL || self.webkitURL const useLoadImageSource = ( options: GLTFParserOptions, - json: GLTF.IGLTF, sourceIndex?: number, loader?: ImageLoader | ImageBitmapLoader ) => { + const json = options.document const result = useHookstate(null) /** @todo caching */ @@ -730,7 +729,7 @@ const useLoadImageSource = ( const sourceURI = useHookstate(null as null | string) let isObjectURL = false - const bufferViewSourceURI = GLTFLoaderFunctions.useLoadBufferView(options, json, sourceDef?.bufferView) + const bufferViewSourceURI = GLTFLoaderFunctions.useLoadBufferView(options, sourceDef?.bufferView) useEffect(() => { if (!sourceDef) return diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 570e8a4074..cdd07eb330 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -84,10 +84,7 @@ import { import { EngineState } from '@etherealengine/spatial/src/EngineState' import { Physics } from '@etherealengine/spatial/src/physics/classes/Physics' import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' -import { - MaterialInstanceComponent, - MaterialStateComponent -} from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { MaterialInstanceComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { ATTRIBUTES } from '../assets/loaders/gltf/GLTFConstants' import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' import { assignExtrasToUserData } from '../assets/loaders/gltf/GLTFLoaderFunctions' @@ -99,6 +96,7 @@ import { GLTFComponent } from './GLTFComponent' import { GLTFDocumentState, GLTFModifiedState, GLTFNodeState, GLTFSnapshotAction } from './GLTFDocumentState' import { GLTFExtensions } from './GLTFExtensions' import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' +import { MaterialDefinitionComponent } from './MaterialDefinitionComponent' export const GLTFAssetState = defineState({ name: 'ee.engine.gltf.GLTFAssetState', @@ -510,18 +508,46 @@ const MaterialStateReactor = (props: { index: number; parentUUID: EntityUUID; do const documentState = useMutableState(GLTFDocumentState)[props.documentID] const entity = props.entity - const material = GLTFLoaderFunctions.useLoadMaterial( - getParserOptions(entity), - documentState.get(NO_PROXY) as GLTF.IGLTF, - props.index - ) + const materialDefinition = documentState.materials![props.index]!.get(NO_PROXY) as GLTF.IMaterial + + useLayoutEffect(() => { + if (!materialDefinition) return + return () => { + removeComponent(entity, MaterialDefinitionComponent) + } + }, []) useLayoutEffect(() => { - if (!entity || !material) return - const uuid = getComponent(entity, UUIDComponent) - material.uuid = uuid - setComponent(entity, MaterialStateComponent, { material }) - }, [material]) + if (!materialDefinition) return + setComponent(entity, MaterialDefinitionComponent, materialDefinition) + }, [materialDefinition]) + + return ( + <> + {materialDefinition.extensions && + Object.entries(materialDefinition.extensions).map(([id, ext]) => { + return + })} + + ) +} + +const MaterialStateExtensionReactor = (props: { keyName: string; value: any; entity: Entity }) => { + const entity = props.entity + + useEffect(() => { + const Component = ComponentJSONIDMap.get(props.keyName) + if (!Component) return console.warn('No component found for extension', props.keyName) + return () => { + removeComponent(entity, Component) + } + }, []) + + useEffect(() => { + const Component = ComponentJSONIDMap.get(props.keyName) + if (!Component) return console.warn('No component found for extension', props.keyName) + setComponent(entity, Component, props.value) + }, [props.keyName, props.value]) return null } @@ -860,11 +886,7 @@ const PrimitiveAttributeReactor = (props: { // Skip attributes already provided by e.g. Draco extension. const attribute = attributeAlreadyLoaded ? undefined : primitive.attributes[props.attribute] - const accessor = GLTFLoaderFunctions.useLoadAccessor( - getParserOptions(props.entity), - documentState.get(NO_PROXY) as GLTF.IGLTF, - attribute - ) + const accessor = GLTFLoaderFunctions.useLoadAccessor(getParserOptions(props.entity), attribute) useEffect(() => { if (!accessor) return @@ -891,11 +913,7 @@ const PrimitiveIndicesAttributeReactor = (props: { const primitive = mesh.primitives[props.primitiveIndex] - const accessor = GLTFLoaderFunctions.useLoadAccessor( - getParserOptions(props.entity), - documentState.get(NO_PROXY) as GLTF.IGLTF, - primitive.indices! - ) + const accessor = GLTFLoaderFunctions.useLoadAccessor(getParserOptions(props.entity), primitive.indices!) useEffect(() => { if (!accessor) return @@ -938,11 +956,13 @@ const MaterialInstanceReactor = (props: { * - we pretty much have to add a new API for each dependency type, like how the GLTFLoader does */ -const getParserOptions = (entity: Entity) => { +export const getParserOptions = (entity: Entity) => { const gltfEntity = getAncestorWithComponent(entity, GLTFComponent) + const document = getState(GLTFDocumentState)[getComponent(gltfEntity, SourceComponent)] const gltfComponent = getComponent(gltfEntity, GLTFComponent) const gltfLoader = getState(AssetLoaderState).gltfLoader return { + document, url: gltfComponent.src, path: LoaderUtils.extractUrlBase(gltfComponent.src), body: gltfComponent.body, diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx new file mode 100644 index 0000000000..efe7db1c17 --- /dev/null +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -0,0 +1,811 @@ +import { + ComponentType, + UUIDComponent, + defineComponent, + getComponent, + setComponent, + useComponent, + useEntityContext +} from '@etherealengine/ecs' +import { NO_PROXY } from '@etherealengine/hyperflux' +import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { GLTF } from '@gltf-transform/core' +import { useEffect, useLayoutEffect } from 'react' +import { Color, LinearSRGBColorSpace, MeshPhysicalMaterial, MeshStandardMaterial, SRGBColorSpace, Vector2 } from 'three' +import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' +import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' +import { getParserOptions } from './GLTFState' + +export const MaterialDefinitionComponent = defineComponent({ + name: 'MaterialDefinitionComponent', + onInit: (entity) => { + return { + type: 'standard' + } as GLTF.IMaterial & { + type: 'standard' | 'basic' | 'physical' + } + }, + + onSet: (entity, component, json) => { + if (!json) return + if (typeof json.type === 'string') component.type.set(json.type) + if (typeof json.pbrMetallicRoughness === 'object') component.pbrMetallicRoughness.set(json.pbrMetallicRoughness) + if (typeof json.normalTexture === 'object') component.normalTexture.set(json.normalTexture) + if (typeof json.occlusionTexture === 'object') component.occlusionTexture.set(json.occlusionTexture) + if (typeof json.emissiveTexture === 'object') component.emissiveTexture.set(json.emissiveTexture) + if (typeof json.emissiveFactor === 'object') component.emissiveFactor.set(json.emissiveFactor) + if (typeof json.alphaMode === 'string') component.alphaMode.set(json.alphaMode) + if (typeof json.alphaCutoff === 'number') component.alphaCutoff.set(json.alphaCutoff) + if (typeof json.doubleSided === 'boolean') component.doubleSided.set(json.doubleSided) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, MaterialDefinitionComponent) + const material = GLTFLoaderFunctions.useLoadMaterial( + getParserOptions(entity), + component.get(NO_PROXY) as ComponentType + ) + + useLayoutEffect(() => { + if (!entity || !material) return + const uuid = getComponent(entity, UUIDComponent) + material.uuid = uuid + setComponent(entity, MaterialStateComponent, { material }) + }, [material]) + + return null + } +}) + +declare module 'three/src/materials/MeshPhysicalMaterial' { + export interface MeshPhysicalMaterial { + setValues(parameters: MeshPhysicalMaterialParameters): void + } +} + +/** + * Unlit Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_unlit + */ +export const GLTFKHRUnlitExtensionComponent = defineComponent({ + name: 'GLTFKHRUnlitExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_UNLIT, + + onInit(entity) { + return {} + }, + + toJSON(entity, component) { + return {} + }, + + reactor: () => { + const entity = useEntityContext() + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'basic' }) + }, []) + + return null + } +}) + +/** + * Materials Emissive Strength Extension + * + * Specification: https://github.com/KhronosGroup/glTF/blob/5768b3ce0ef32bc39cdf1bef10b948586635ead3/extensions/2.0/Khronos/KHR_materials_emissive_strength/README.md + */ +export const GLTFKHREmissiveStrengthExtensionComponent = defineComponent({ + name: 'GLTFKHREmissiveStrengthExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_EMISSIVE_STRENGTH, + + onInit(entity) { + return {} as { + emissiveStrength?: number + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.emissiveStrength === 'number') component.emissiveStrength.set(json.emissiveStrength) + }, + + toJSON(entity, component) { + return { + emissiveStrength: component.emissiveStrength.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHREmissiveStrengthExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + if (typeof component.emissiveStrength.value !== 'number') return + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ emissiveIntensity: component.emissiveStrength.value }) + }, [component.emissiveStrength.value]) + + return null + } +}) + +/** + * Clearcoat Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_clearcoat + */ +export const GLTFKHRClearcoatExtensionComponent = defineComponent({ + name: 'GLTFKHRClearcoatExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_CLEARCOAT, + + onInit(entity) { + return {} as { + clearcoatFactor?: number + clearcoatTexture?: GLTF.ITextureInfo + clearcoatRoughnessFactor?: number + clearcoatRoughnessTexture?: GLTF.ITextureInfo + clearcoatNormalTexture?: GLTF.IMaterialNormalTextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.clearcoatFactor === 'number') component.clearcoatFactor.set(json.clearcoatFactor) + if (typeof json.clearcoatTexture === 'object') component.clearcoatTexture.set(json.clearcoatTexture) + if (typeof json.clearcoatRoughnessFactor === 'number') + component.clearcoatRoughnessFactor.set(json.clearcoatRoughnessFactor) + if (typeof json.clearcoatRoughnessTexture === 'object') + component.clearcoatRoughnessTexture.set(json.clearcoatRoughnessTexture) + if (typeof json.clearcoatNormalTexture === 'object') + component.clearcoatNormalTexture.set(json.clearcoatNormalTexture) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRClearcoatExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + }, []) + + useEffect(() => { + if (!component.clearcoatFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ clearcoat: component.clearcoatFactor.value }) + material.needsUpdate = true + }, [component.clearcoatFactor.value]) + + useEffect(() => { + if (!component.clearcoatRoughnessFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ clearcoatRoughness: component.clearcoatRoughnessFactor.value }) + material.needsUpdate = true + }, [component.clearcoatRoughnessFactor.value]) + + const options = getParserOptions(entity) + const clearcoatMap = GLTFLoaderFunctions.useAssignTexture(options, component.clearcoatTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ clearcoatMap }) + material.needsUpdate = true + }, [clearcoatMap]) + + const clearcoatRoughnessMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.clearcoatRoughnessTexture.value + ) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ clearcoatRoughnessMap }) + material.needsUpdate = true + }, [clearcoatRoughnessMap]) + + const clearcoatNormalMap = GLTFLoaderFunctions.useAssignTexture(options, component.clearcoatNormalTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + + if (component.clearcoatNormalTexture.value?.scale !== undefined) { + const scale = component.clearcoatNormalTexture.value.scale + material.setValues({ clearcoatNormalScale: new Vector2(scale, scale) }) + } + + material.setValues({ clearcoatNormalMap }) + material.needsUpdate = true + }, [clearcoatNormalMap]) + + return null + } +}) + +/** + * Iridescence Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_iridescence + */ +export const GLTFKHRIridescenceExtensionComponent = defineComponent({ + name: 'GLTFKHRIridescenceExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_IRIDESCENCE, + + onInit(entity) { + return {} as { + iridescenceFactor?: number + iridescenceTexture?: GLTF.ITextureInfo + iridescenceIor?: number + iridescenceThicknessMinimum?: number + iridescenceThicknessMaximum?: number + iridescenceThicknessTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.iridescenceFactor === 'number') component.iridescenceFactor.set(json.iridescenceFactor) + if (typeof json.iridescenceTexture === 'object') component.iridescenceTexture.set(json.iridescenceTexture) + if (typeof json.iridescenceIor === 'number') component.iridescenceIor.set(json.iridescenceIor) + if (typeof json.iridescenceThicknessMinimum === 'number') + component.iridescenceThicknessMinimum.set(json.iridescenceThicknessMinimum) + if (typeof json.iridescenceThicknessMaximum === 'number') + component.iridescenceThicknessMaximum.set(json.iridescenceThicknessMaximum) + if (typeof json.iridescenceThicknessTexture === 'object') + component.iridescenceThicknessTexture.set(json.iridescenceThicknessTexture) + }, + + toJSON(entity, component) { + return { + iridescenceFactor: component.iridescenceFactor.value, + iridescenceTexture: component.iridescenceTexture.value, + iridescenceIor: component.iridescenceIor.value, + iridescenceThicknessMinimum: component.iridescenceThicknessMinimum.value, + iridescenceThicknessMaximum: component.iridescenceThicknessMaximum.value, + iridescenceThicknessTexture: component.iridescenceThicknessTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRIridescenceExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + }, []) + + useEffect(() => { + if (!component.iridescenceFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ iridescence: component.iridescenceFactor.value }) + material.needsUpdate = true + }, [component.iridescenceFactor.value]) + + useEffect(() => { + if (!component.iridescenceIor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ iridescenceIOR: component.iridescenceIor.value }) + material.needsUpdate = true + }, [component.iridescenceIor.value]) + + const options = getParserOptions(entity) + const iridescenceMap = GLTFLoaderFunctions.useAssignTexture(options, component.iridescenceTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ iridescenceMap }) + material.needsUpdate = true + }, [iridescenceMap]) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ + iridescenceThicknessRange: [ + component.iridescenceThicknessMinimum.value ?? 100, + component.iridescenceThicknessMaximum.value ?? 400 + ] + }) + material.needsUpdate = true + }, [component.iridescenceThicknessMinimum.value, component.iridescenceThicknessMaximum.value]) + + const iridescenceThicknessMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.iridescenceThicknessTexture.value + ) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ iridescenceThicknessMap }) + material.needsUpdate = true + }, [iridescenceThicknessMap]) + + return null + } +}) + +/** + * Sheen Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_materials_sheen + */ +export const GLTFKHRSheenExtensionComponent = defineComponent({ + name: 'GLTFKHRSheenExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_SHEEN, + + onInit(entity) { + return {} as { + sheenColorFactor?: [number, number, number] + sheenRoughnessFactor?: number + sheenColorTexture?: GLTF.ITextureInfo + sheenRoughnessTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (Array.isArray(json.sheenColorFactor)) component.sheenColorFactor.set(json.sheenColorFactor) + if (typeof json.sheenRoughnessFactor === 'number') component.sheenRoughnessFactor.set(json.sheenRoughnessFactor) + if (typeof json.sheenColorTexture === 'object') component.sheenColorTexture.set(json.sheenColorTexture) + if (typeof json.sheenRoughnessTexture === 'object') component.sheenRoughnessTexture.set(json.sheenRoughnessTexture) + }, + + toJSON(entity, component) { + return { + sheenColorFactor: component.sheenColorFactor.value, + sheenRoughnessFactor: component.sheenRoughnessFactor.value, + sheenColorTexture: component.sheenColorTexture.value, + sheenRoughnessTexture: component.sheenRoughnessTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRSheenExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ sheen: 1 }) + }, []) + + useEffect(() => { + if (!component.sheenColorFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ + sheenColor: new Color().setRGB( + component.sheenColorFactor.value[0], + component.sheenColorFactor.value[1], + component.sheenColorFactor.value[2], + LinearSRGBColorSpace + ) + }) + material.needsUpdate = true + }, [component.sheenColorFactor.value]) + + useEffect(() => { + if (!component.sheenRoughnessFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ sheenRoughness: component.sheenRoughnessFactor.value }) + material.needsUpdate = true + }, [component.sheenRoughnessFactor.value]) + + const options = getParserOptions(entity) + const sheenColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.sheenColorTexture.value) + + useEffect(() => { + if (sheenColorMap) sheenColorMap.colorSpace = SRGBColorSpace + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ sheenColorMap }) + material.needsUpdate = true + }, [sheenColorMap]) + + const sheenRoughnessMap = GLTFLoaderFunctions.useAssignTexture(options, component.sheenRoughnessTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ sheenRoughnessMap }) + material.needsUpdate = true + }, [sheenRoughnessMap]) + + return null + } +}) + +/** + * Transmission Materials Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_transmission + * Draft: https://github.com/KhronosGroup/glTF/pull/1698 + */ +export const GLTFKHRTransmissionExtensionComponent = defineComponent({ + name: 'GLTFKHRTransmissionExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_TRANSMISSION, + + onInit(entity) { + return {} as { + transmissionFactor?: number + transmissionTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.transmissionFactor === 'number') component.transmissionFactor.set(json.transmissionFactor) + if (typeof json.transmissionTexture === 'object') component.transmissionTexture.set(json.transmissionTexture) + }, + + toJSON(entity, component) { + return { + transmissionFactor: component.transmissionFactor.value, + transmissionTexture: component.transmissionTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRTransmissionExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + }, []) + + useEffect(() => { + if (!component.transmissionFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ transmission: component.transmissionFactor.value }) + material.needsUpdate = true + }, [component.transmissionFactor.value]) + + const options = getParserOptions(entity) + const transmissionMap = GLTFLoaderFunctions.useAssignTexture(options, component.transmissionTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ transmissionMap }) + material.needsUpdate = true + }, [transmissionMap]) + + return null + } +}) + +/** + * Materials Volume Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_volume + */ +export const GLTFKHRVolumeExtensionComponent = defineComponent({ + name: 'GLTFKHRVolumeExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_VOLUME, + + onInit(entity) { + return {} as { + thicknessFactor?: number + thicknessTexture?: GLTF.ITextureInfo + attenuationDistance?: number + attenuationColorFactor?: [number, number, number] + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.thicknessFactor === 'number') component.thicknessFactor.set(json.thicknessFactor) + if (typeof json.thicknessTexture === 'object') component.thicknessTexture.set(json.thicknessTexture) + if (typeof json.attenuationDistance === 'number') component.attenuationDistance.set(json.attenuationDistance) + if (Array.isArray(json.attenuationColorFactor)) component.attenuationColorFactor.set(json.attenuationColorFactor) + }, + + toJSON(entity, component) { + return { + thicknessFactor: component.thicknessFactor.value, + thicknessTexture: component.thicknessTexture.value, + attenuationDistance: component.attenuationDistance.value, + attenuationColorFactor: component.attenuationColorFactor.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRVolumeExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ thickness: component.thicknessFactor.value ?? 0 }) + material.needsUpdate = true + }, [component.thicknessFactor.value]) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ attenuationDistance: component.attenuationDistance.value || Infinity }) + material.needsUpdate = true + }, [component.attenuationDistance.value]) + + useEffect(() => { + if (!component.attenuationColorFactor.value) return + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ + attenuationColor: new Color().setRGB( + component.attenuationColorFactor.value[0], + component.attenuationColorFactor.value[1], + component.attenuationColorFactor.value[2], + LinearSRGBColorSpace + ) + }) + material.needsUpdate = true + }, [component.attenuationColorFactor.value]) + + const options = getParserOptions(entity) + const thicknessMap = GLTFLoaderFunctions.useAssignTexture(options, component.thicknessTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ thicknessMap }) + material.needsUpdate = true + }, [thicknessMap]) + + return null + } +}) + +/** + * Materials ior Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_ior + */ +export const GLTFKHRIorExtensionComponent = defineComponent({ + name: 'GLTFKHRIorExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_IOR, + + onInit(entity) { + return {} as { + ior?: number + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.ior === 'number') component.ior.set(json.ior) + }, + + toJSON(entity, component) { + return { + ior: component.ior.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRIorExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ ior: component.ior.value ?? 1.5 }) + material.needsUpdate = true + }, [component.ior.value]) + + return null + } +}) + +/** + * Materials specular Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_specular + */ +export const GLTFKHRSpecularExtensionComponent = defineComponent({ + name: 'GLTFKHRSpecularExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_SPECULAR, + + onInit(entity) { + return {} as { + specularFactor?: number + specularTexture?: GLTF.ITextureInfo + specularColorFactor?: [number, number, number] + specularColorTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.specularFactor === 'number') component.specularFactor.set(json.specularFactor) + if (typeof json.specularTexture === 'object') component.specularTexture.set(json.specularTexture) + if (Array.isArray(json.specularColorFactor)) component.specularColorFactor.set(json.specularColorFactor) + if (typeof json.specularColorTexture === 'object') component.specularColorTexture.set(json.specularColorTexture) + }, + + toJSON(entity, component) { + return { + specularFactor: component.specularFactor.value, + specularTexture: component.specularTexture.value, + specularColorFactor: component.specularColorFactor.value, + specularColorTexture: component.specularColorTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRSpecularExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ specularIntensity: component.specularFactor.value ?? 1.0 }) + material.needsUpdate = true + }, [component.specularFactor.value]) + + useEffect(() => { + const specularColorFactor = component.specularColorFactor.value ?? [1, 1, 1] + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ + specularColor: new Color().setRGB( + specularColorFactor[0], + specularColorFactor[1], + specularColorFactor[2], + LinearSRGBColorSpace + ) + }) + material.needsUpdate = true + }, [component.specularColorFactor.value]) + + const options = getParserOptions(entity) + const specularMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ specularIntensityMap: specularMap }) + }, [specularMap]) + + const specularColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularColorTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ specularColorMap }) + material.needsUpdate = true + }, [specularColorMap]) + + return null + } +}) + +/** + * Materials bump Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/EXT_materials_bump + */ +export const GLTFKHRBumpExtensionComponent = defineComponent({ + name: 'GLTFKHRBumpExtensionComponent', + jsonID: EXTENSIONS.EXT_MATERIALS_BUMP, + + onInit(entity) { + return {} as { + bumpFactor?: number + bumpTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.bumpFactor === 'number') component.bumpFactor.set(json.bumpFactor) + if (typeof json.bumpTexture === 'object') component.bumpTexture.set(json.bumpTexture) + }, + + toJSON(entity, component) { + return { + bumpFactor: component.bumpFactor.value, + bumpTexture: component.bumpTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRBumpExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ bumpScale: component.bumpFactor.value ?? 1.0 }) + material.needsUpdate = true + }, [component.bumpFactor.value]) + + const options = getParserOptions(entity) + const bumpMap = GLTFLoaderFunctions.useAssignTexture(options, component.bumpTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ bumpMap }) + material.needsUpdate = true + }, [bumpMap]) + + return null + } +}) + +/** + * Materials anisotropy Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_anisotropy + */ +export const GLTFKHRAnisotropyExtensionComponent = defineComponent({ + name: 'GLTFKHRAnisotropyExtensionComponent', + jsonID: EXTENSIONS.KHR_MATERIALS_ANISOTROPY, + + onInit(entity) { + return {} as { + anisotropyStrength?: number + anisotropyRotation?: number + anisotropyTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.anisotropyStrength === 'number') component.anisotropyStrength.set(json.anisotropyStrength) + if (typeof json.anisotropyRotation === 'number') component.anisotropyRotation.set(json.anisotropyRotation) + if (typeof json.anisotropyTexture === 'object') component.anisotropyTexture.set(json.anisotropyTexture) + }, + + toJSON(entity, component) { + return { + anisotropyStrength: component.anisotropyStrength.value, + anisotropyRotation: component.anisotropyRotation.value, + anisotropyTexture: component.anisotropyTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, GLTFKHRAnisotropyExtensionComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ anisotropy: component.anisotropyStrength.value ?? 0.0 }) + material.needsUpdate = true + }, [component.anisotropyStrength.value]) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ anisotropyRotation: component.anisotropyRotation.value ?? 0.0 }) + material.needsUpdate = true + }, [component.anisotropyRotation.value]) + + const options = getParserOptions(entity) + const anisotropyMap = GLTFLoaderFunctions.useAssignTexture(options, component.anisotropyTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + material.setValues({ anisotropyMap }) + material.needsUpdate = true + }, [anisotropyMap]) + + return null + } +}) From 53dc8340dc8beef052c7861043fa47288d4f8192 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 13:01:59 +1000 Subject: [PATCH 15/47] license --- .../src/gltf/MaterialDefinitionComponent.tsx | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx index efe7db1c17..29f7917969 100644 --- a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -1,3 +1,28 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + import { ComponentType, UUIDComponent, From 625dda0063a3ed476f151a90952ccbbc11dc274d Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 13:43:03 +1000 Subject: [PATCH 16/47] bug fix --- packages/engine/src/gltf/GLTFLoaderFunctions.ts | 7 +------ packages/engine/src/gltf/MaterialDefinitionComponent.tsx | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 3aeeddd235..36dcb4dac1 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -560,7 +560,6 @@ const useLoadMaterial = ( * @return {Promise} */ const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo) => { - const json = options.document const result = useHookstate(null) const texture = GLTFLoaderFunctions.useLoadTexture(options, mapDef?.index) @@ -604,7 +603,6 @@ const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo */ const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { const json = options.document - const result = useHookstate(null) const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null const sourceIndex = textureDef?.source! @@ -646,11 +644,8 @@ const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { } const texture = GLTFLoaderFunctions.useLoadTextureImage(options, textureIndex, sourceIndex, loader) - useEffect(() => { - result.set(texture) - }, [texture]) - return result.value as Texture | null + return texture } const textureCache = {} as any // todo diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx index 29f7917969..9dd63c6f6f 100644 --- a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -698,6 +698,7 @@ export const GLTFKHRSpecularExtensionComponent = defineComponent({ useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ specularIntensityMap: specularMap }) + material.needsUpdate = true }, [specularMap]) const specularColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularColorTexture.value) From 1e55e9450c103eab833751af08ffe5c0714f92bc Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 14:29:29 +1000 Subject: [PATCH 17/47] add punctual lights extension --- packages/engine/src/gltf/GLTFState.tsx | 1 + .../src/gltf/MaterialDefinitionComponent.tsx | 64 +++++----- .../src/gltf/MeshExtensionComponents.ts | 118 ++++++++++++++++++ 3 files changed, 151 insertions(+), 32 deletions(-) create mode 100644 packages/engine/src/gltf/MeshExtensionComponents.ts diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index cdd07eb330..5fedf358db 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -97,6 +97,7 @@ import { GLTFDocumentState, GLTFModifiedState, GLTFNodeState, GLTFSnapshotAction import { GLTFExtensions } from './GLTFExtensions' import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' import { MaterialDefinitionComponent } from './MaterialDefinitionComponent' +import './MeshExtensionComponents' export const GLTFAssetState = defineState({ name: 'ee.engine.gltf.GLTFAssetState', diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx index 9dd63c6f6f..0f211147c8 100644 --- a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -94,8 +94,8 @@ declare module 'three/src/materials/MeshPhysicalMaterial' { * * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_unlit */ -export const GLTFKHRUnlitExtensionComponent = defineComponent({ - name: 'GLTFKHRUnlitExtensionComponent', +export const KHRUnlitExtensionComponent = defineComponent({ + name: 'KHRUnlitExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_UNLIT, onInit(entity) { @@ -122,8 +122,8 @@ export const GLTFKHRUnlitExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/blob/5768b3ce0ef32bc39cdf1bef10b948586635ead3/extensions/2.0/Khronos/KHR_materials_emissive_strength/README.md */ -export const GLTFKHREmissiveStrengthExtensionComponent = defineComponent({ - name: 'GLTFKHREmissiveStrengthExtensionComponent', +export const KHREmissiveStrengthExtensionComponent = defineComponent({ + name: 'KHREmissiveStrengthExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_EMISSIVE_STRENGTH, onInit(entity) { @@ -145,7 +145,7 @@ export const GLTFKHREmissiveStrengthExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHREmissiveStrengthExtensionComponent) + const component = useComponent(entity, KHREmissiveStrengthExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -163,8 +163,8 @@ export const GLTFKHREmissiveStrengthExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_clearcoat */ -export const GLTFKHRClearcoatExtensionComponent = defineComponent({ - name: 'GLTFKHRClearcoatExtensionComponent', +export const KHRClearcoatExtensionComponent = defineComponent({ + name: 'KHRClearcoatExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_CLEARCOAT, onInit(entity) { @@ -191,7 +191,7 @@ export const GLTFKHRClearcoatExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRClearcoatExtensionComponent) + const component = useComponent(entity, KHRClearcoatExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -255,8 +255,8 @@ export const GLTFKHRClearcoatExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_iridescence */ -export const GLTFKHRIridescenceExtensionComponent = defineComponent({ - name: 'GLTFKHRIridescenceExtensionComponent', +export const KHRIridescenceExtensionComponent = defineComponent({ + name: 'KHRIridescenceExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_IRIDESCENCE, onInit(entity) { @@ -296,7 +296,7 @@ export const GLTFKHRIridescenceExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRIridescenceExtensionComponent) + const component = useComponent(entity, KHRIridescenceExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -357,8 +357,8 @@ export const GLTFKHRIridescenceExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_materials_sheen */ -export const GLTFKHRSheenExtensionComponent = defineComponent({ - name: 'GLTFKHRSheenExtensionComponent', +export const KHRSheenExtensionComponent = defineComponent({ + name: 'KHRSheenExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_SHEEN, onInit(entity) { @@ -389,7 +389,7 @@ export const GLTFKHRSheenExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRSheenExtensionComponent) + const component = useComponent(entity, KHRSheenExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -447,8 +447,8 @@ export const GLTFKHRSheenExtensionComponent = defineComponent({ * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_transmission * Draft: https://github.com/KhronosGroup/glTF/pull/1698 */ -export const GLTFKHRTransmissionExtensionComponent = defineComponent({ - name: 'GLTFKHRTransmissionExtensionComponent', +export const KHRTransmissionExtensionComponent = defineComponent({ + name: 'KHRTransmissionExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_TRANSMISSION, onInit(entity) { @@ -473,7 +473,7 @@ export const GLTFKHRTransmissionExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRTransmissionExtensionComponent) + const component = useComponent(entity, KHRTransmissionExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -505,8 +505,8 @@ export const GLTFKHRTransmissionExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_volume */ -export const GLTFKHRVolumeExtensionComponent = defineComponent({ - name: 'GLTFKHRVolumeExtensionComponent', +export const KHRVolumeExtensionComponent = defineComponent({ + name: 'KHRVolumeExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_VOLUME, onInit(entity) { @@ -537,7 +537,7 @@ export const GLTFKHRVolumeExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRVolumeExtensionComponent) + const component = useComponent(entity, KHRVolumeExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -588,8 +588,8 @@ export const GLTFKHRVolumeExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_ior */ -export const GLTFKHRIorExtensionComponent = defineComponent({ - name: 'GLTFKHRIorExtensionComponent', +export const KHRIorExtensionComponent = defineComponent({ + name: 'KHRIorExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_IOR, onInit(entity) { @@ -611,7 +611,7 @@ export const GLTFKHRIorExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRIorExtensionComponent) + const component = useComponent(entity, KHRIorExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -633,8 +633,8 @@ export const GLTFKHRIorExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_specular */ -export const GLTFKHRSpecularExtensionComponent = defineComponent({ - name: 'GLTFKHRSpecularExtensionComponent', +export const KHRSpecularExtensionComponent = defineComponent({ + name: 'KHRSpecularExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_SPECULAR, onInit(entity) { @@ -665,7 +665,7 @@ export const GLTFKHRSpecularExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRSpecularExtensionComponent) + const component = useComponent(entity, KHRSpecularExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -718,8 +718,8 @@ export const GLTFKHRSpecularExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/EXT_materials_bump */ -export const GLTFKHRBumpExtensionComponent = defineComponent({ - name: 'GLTFKHRBumpExtensionComponent', +export const EXTBumpExtensionComponent = defineComponent({ + name: 'EXTBumpExtensionComponent', jsonID: EXTENSIONS.EXT_MATERIALS_BUMP, onInit(entity) { @@ -744,7 +744,7 @@ export const GLTFKHRBumpExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRBumpExtensionComponent) + const component = useComponent(entity, EXTBumpExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { @@ -775,8 +775,8 @@ export const GLTFKHRBumpExtensionComponent = defineComponent({ * * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_materials_anisotropy */ -export const GLTFKHRAnisotropyExtensionComponent = defineComponent({ - name: 'GLTFKHRAnisotropyExtensionComponent', +export const KHRAnisotropyExtensionComponent = defineComponent({ + name: 'KHRAnisotropyExtensionComponent', jsonID: EXTENSIONS.KHR_MATERIALS_ANISOTROPY, onInit(entity) { @@ -804,7 +804,7 @@ export const GLTFKHRAnisotropyExtensionComponent = defineComponent({ reactor: () => { const entity = useEntityContext() - const component = useComponent(entity, GLTFKHRAnisotropyExtensionComponent) + const component = useComponent(entity, KHRAnisotropyExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { diff --git a/packages/engine/src/gltf/MeshExtensionComponents.ts b/packages/engine/src/gltf/MeshExtensionComponents.ts new file mode 100644 index 0000000000..2876c8c548 --- /dev/null +++ b/packages/engine/src/gltf/MeshExtensionComponents.ts @@ -0,0 +1,118 @@ +import { defineComponent, removeComponent, setComponent, useComponent, useEntityContext } from '@etherealengine/ecs' +import { DirectionalLightComponent, PointLightComponent, SpotLightComponent } from '@etherealengine/spatial' +import { useEffect } from 'react' +import { Color } from 'three' +import { getParserOptions } from './GLTFState' + +export type KHRPunctualLight = { + color?: [number, number, number] + intensity?: number + range?: number + type: 'directional' | 'point' | 'spot' + spot?: { + innerConeAngle?: number + outerConeAngle?: number + } +} + +export const KHRLightsPunctualComponent = defineComponent({ + name: 'KHRLightsPunctualComponent', + jsonID: 'KHR_lights_punctual', + + onInit(entity) { + return {} as { + light?: number + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.light === 'number') component.light.set(json.light) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRLightsPunctualComponent) + + const options = getParserOptions(entity) + const json = options.document + const extensions: { + lights?: KHRPunctualLight[] + } = (json.extensions && json.extensions[KHRLightsPunctualComponent.jsonID]) || {} + const lightDefs = extensions.lights + const lightDef = lightDefs && component.light.value ? lightDefs[component.light.value] : undefined + + useEffect(() => { + return () => { + removeComponent(entity, DirectionalLightComponent) + removeComponent(entity, SpotLightComponent) + removeComponent(entity, PointLightComponent) + } + }, [lightDef?.type]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'directional') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + + setComponent(entity, DirectionalLightComponent, { + color, + intensity + }) + }, [lightDef]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'spot') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + const range = typeof lightDef.range === 'number' ? lightDef.range : undefined + const innerConeAngle = typeof lightDef.spot?.innerConeAngle === 'number' ? lightDef.spot.innerConeAngle : 0 + const outerConeAngle = + typeof lightDef.spot?.outerConeAngle === 'number' ? lightDef.spot.outerConeAngle : Math.PI / 4.0 + + const penumbra = 1.0 - innerConeAngle / outerConeAngle + const angle = outerConeAngle + + setComponent(entity, SpotLightComponent, { + color, + intensity, + decay: 2, + range, + angle, + penumbra + }) + }, [lightDef]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'point') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + const range = typeof lightDef.range === 'number' ? lightDef.range : undefined + + setComponent(entity, PointLightComponent, { + color, + intensity, + decay: 2, + range + }) + }, [lightDef]) + + return null + } +}) From 358a046795e73a777d4ae3c34ddc46d67bb7af15 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 14:29:42 +1000 Subject: [PATCH 18/47] license --- .../src/gltf/MeshExtensionComponents.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/engine/src/gltf/MeshExtensionComponents.ts b/packages/engine/src/gltf/MeshExtensionComponents.ts index 2876c8c548..16113b1cd3 100644 --- a/packages/engine/src/gltf/MeshExtensionComponents.ts +++ b/packages/engine/src/gltf/MeshExtensionComponents.ts @@ -1,3 +1,28 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + import { defineComponent, removeComponent, setComponent, useComponent, useEntityContext } from '@etherealengine/ecs' import { DirectionalLightComponent, PointLightComponent, SpotLightComponent } from '@etherealengine/spatial' import { useEffect } from 'react' From 4d4f64c01ea1cdf2c799ce96ac32039770420109 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 15:43:21 +1000 Subject: [PATCH 19/47] add camera --- packages/engine/src/gltf/GLTFState.tsx | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 5fedf358db..9b7c2f2a13 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -81,6 +81,7 @@ import { getAncestorWithComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { CameraComponent } from '@etherealengine/spatial/src/camera/components/CameraComponent' import { EngineState } from '@etherealengine/spatial/src/EngineState' import { Physics } from '@etherealengine/spatial/src/physics/classes/Physics' import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' @@ -696,6 +697,9 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: {typeof node.mesh.get(NO_PROXY) === 'number' && ( )} + {typeof node.camera.get(NO_PROXY) === 'number' && ( + + )} {node.extensions.value && Object.keys(node.extensions.get(NO_PROXY)!).map((extension) => ( { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const camera = documentState.cameras.get(NO_PROXY)![node.camera!] as GLTF.ICamera + + useEffect(() => { + if (camera.type === 'orthographic' || !camera.perspective) + return console.warn('Orthographic cameras not supported yet') + + const perspectiveCamera = camera.perspective + + setComponent(props.entity, CameraComponent, { + fov: MathUtils.radToDeg(perspectiveCamera.yfov), + aspect: perspectiveCamera.aspectRatio || 1, + near: perspectiveCamera.znear || 1, + far: perspectiveCamera.zfar || 2e6 + }) + }, [camera]) + + return null +} + const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; documentID: string; entity: Entity }) => { const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) From 33445965e4ba112d6e9c49eada53f225acf6448c Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 17:22:30 +1000 Subject: [PATCH 20/47] fix draco --- packages/engine/src/gltf/GLTFExtensions.ts | 108 ++++++------------ .../engine/src/gltf/GLTFLoaderFunctions.ts | 54 ++------- packages/engine/src/gltf/GLTFState.tsx | 49 ++++++-- 3 files changed, 85 insertions(+), 126 deletions(-) diff --git a/packages/engine/src/gltf/GLTFExtensions.ts b/packages/engine/src/gltf/GLTFExtensions.ts index 81dce835a4..88f65111b0 100644 --- a/packages/engine/src/gltf/GLTFExtensions.ts +++ b/packages/engine/src/gltf/GLTFExtensions.ts @@ -23,25 +23,19 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { getState } from '@etherealengine/hyperflux' +import { getState, startReactor } from '@etherealengine/hyperflux' import { GLTF } from '@gltf-transform/core' -import { - BufferGeometry, - Color, - LinearSRGBColorSpace, - MeshBasicMaterial, - NormalBufferAttributes, - SRGBColorSpace -} from 'three' +import { useEffect } from 'react' +import { BufferGeometry, NormalBufferAttributes } from 'three' import { ATTRIBUTES, WEBGL_COMPONENT_TYPES } from '../assets/loaders/gltf/GLTFConstants' import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' import { AssetLoaderState } from '../assets/state/AssetLoaderState' import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' -const KHR_DRACO_MESH_COMPRESSION = { - decodePrimitive(options: GLTFParserOptions, json: GLTF.IGLTF, primitive: GLTF.IMeshPrimitive) { - const dracoLoader = getState(AssetLoaderState).gltfLoader.dracoLoader! +export const KHR_DRACO_MESH_COMPRESSION = { + decodePrimitive(options: GLTFParserOptions, primitive: GLTF.IMeshPrimitive) { + const json = options.document const dracoMeshCompressionExtension = primitive.extensions![EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] as any const bufferViewIndex = dracoMeshCompressionExtension.bufferView const gltfAttributeMap = dracoMeshCompressionExtension.attributes @@ -59,8 +53,7 @@ const KHR_DRACO_MESH_COMPRESSION = { const threeAttributeName = ATTRIBUTES[attributeName] || attributeName.toLowerCase() if (gltfAttributeMap[attributeName] !== undefined) { - // @ts-ignore -- TODO type extensions - const accessorDef = json.accessors[primitive.attributes[attributeName]] + const accessorDef = json.accessors![primitive.attributes[attributeName]] const componentType = WEBGL_COMPONENT_TYPES[accessorDef.componentType] attributeTypeMap[threeAttributeName] = componentType.name @@ -68,68 +61,33 @@ const KHR_DRACO_MESH_COMPRESSION = { } } - return GLTFLoaderFunctions.loadBufferView(options, json, bufferViewIndex).then(function (bufferView) { - return new Promise>(function (resolve) { - dracoLoader.preload().decodeDracoFile( - bufferView, - function (geometry) { - for (const attributeName in geometry.attributes) { - const attribute = geometry.attributes[attributeName] - const normalized = attributeNormalizedMap[attributeName] - - if (normalized !== undefined) attribute.normalized = normalized - } - - resolve(geometry) - }, - threeAttributeMap, - attributeTypeMap - ) + return new Promise>(function (resolve) { + /** + * Using an inline reactor here allows us to use reference counting & resource caching, + * and release the uncompressed buffer as soon as it is no longer required + */ + const reactor = startReactor(() => { + const bufferView = GLTFLoaderFunctions.useLoadBufferView(options, bufferViewIndex) + useEffect(() => { + if (!bufferView) return + const dracoLoader = getState(AssetLoaderState).gltfLoader.dracoLoader! + dracoLoader.preload().decodeDracoFile( + bufferView, + function (geometry) { + for (const attributeName in geometry.attributes) { + const attribute = geometry.attributes[attributeName] + const normalized = attributeNormalizedMap[attributeName] + if (normalized !== undefined) attribute.normalized = normalized + } + resolve(geometry) + reactor.stop() + }, + threeAttributeMap, + attributeTypeMap + ) + }, [bufferView]) + return null }) }) } } - -const KHR_MATERIALS_UNLIT = { - getMaterialType() { - return MeshBasicMaterial - }, - - extendParams(options: GLTFParserOptions, json: GLTF.IGLTF, materialParams, materialDef) { - const pending = [] as Promise[] - - materialParams.color = new Color(1.0, 1.0, 1.0) - materialParams.opacity = 1.0 - - const metallicRoughness = materialDef.pbrMetallicRoughness - - if (metallicRoughness) { - if (Array.isArray(metallicRoughness.baseColorFactor)) { - const array = metallicRoughness.baseColorFactor - - materialParams.color.setRGB(array[0], array[1], array[2], LinearSRGBColorSpace) - materialParams.opacity = array[3] - } - - if (metallicRoughness.baseColorTexture !== undefined) { - pending.push( - GLTFLoaderFunctions.assignTexture( - options, - json, - materialParams, - 'map', - metallicRoughness.baseColorTexture, - SRGBColorSpace - ) - ) - } - } - - return Promise.all(pending) - } -} - -export const GLTFExtensions = { - [EXTENSIONS.KHR_DRACO_MESH_COMPRESSION]: KHR_DRACO_MESH_COMPRESSION, - [EXTENSIONS.KHR_MATERIALS_UNLIT]: KHR_MATERIALS_UNLIT -} diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 36dcb4dac1..0c6941e8b8 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -60,10 +60,8 @@ import { WEBGL_TYPE_SIZES, WEBGL_WRAPPINGS } from '../assets/loaders/gltf/GLTFConstants' -import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' import { assignExtrasToUserData, getNormalizedComponentScale } from '../assets/loaders/gltf/GLTFLoaderFunctions' import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' -import { GLTFExtensions } from './GLTFExtensions' import { MaterialDefinitionComponent } from './MaterialDefinitionComponent' // todo make this a state @@ -237,7 +235,7 @@ const useLoadBuffer = (options: GLTFParserOptions, bufferIndex?: number) => { return } - /** @todo use a global file loader */ + /** @todo use resource hooks */ const fileLoader = new FileLoader(options.manager) fileLoader.setResponseType('arraybuffer') if (options.crossOrigin === 'use-credentials') { @@ -376,33 +374,6 @@ const useLoadMaterial = ( }, [materialDef.type]) const material = result.get(NO_PROXY) as MeshStandardMaterial | MeshBasicMaterial - - const materialExtensions = materialDef.extensions || {} - - /** @todo expose 'getMaterialType' API */ - - // materialType = this._invokeOne(function (ext) { - // return ext.getMaterialType && ext.getMaterialType(materialIndex) - // }) - - /** @todo expose API */ - // pending.push( - // Promise.all( - // this._invokeAll(function (ext) { - // return ext.extendMaterialParams && ext.extendMaterialParams(materialIndex, materialParams) - // }) - // ) - // ) - // } - - // if (!materialExtensions[EXTENSIONS.EE_MATERIAL] && materialExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT]) { - // const kmuExtension = GLTFExtensions[EXTENSIONS.KHR_MATERIALS_UNLIT] - // materialType = kmuExtension.getMaterialType() - // pending.push(kmuExtension.extendParams(options, materialParams, materialDef)) - // } else { - // Specification: - // https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#metallic-roughness-material - const map = GLTFLoaderFunctions.useAssignTexture(options, materialDef.pbrMetallicRoughness?.baseColorTexture) useEffect(() => { @@ -580,17 +551,16 @@ const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo result.set(texture) - if (GLTFExtensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM]) { - const transform = - mapDef.extensions !== undefined ? mapDef.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM] : undefined + // if (GLTFExtensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM]) { + // const transform = + // mapDef.extensions !== undefined ? mapDef.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM] : undefined - if (transform) { - /** @todo */ - // const gltfReference = parser.associations.get(texture) - // texture = parser.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM].extendTexture(texture, transform) - // parser.associations.set(texture, gltfReference) - } - } + // if (transform) { + // const gltfReference = parser.associations.get(texture) + // texture = parser.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM].extendTexture(texture, transform) + // parser.associations.set(texture, gltfReference) + // } + // } }, [texture, mapDef]) return result.value as Texture | null @@ -621,7 +591,7 @@ const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { let textureLoader - /** @todo make global loader */ + /** @todo use resource loader hooks */ if (typeof createImageBitmap === 'undefined' || isSafari || (isFirefox && firefoxVersion < 98)) { textureLoader = new TextureLoader(options.manager) } else { @@ -648,8 +618,6 @@ const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { return texture } -const textureCache = {} as any // todo - const useLoadTextureImage = ( options: GLTFParserOptions, textureIndex?: number, diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 9b7c2f2a13..a31fd3e4fa 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -95,7 +95,7 @@ import { SourceComponent } from '../scene/components/SourceComponent' import { proxifyParentChildRelationships } from '../scene/functions/loadGLTFModel' import { GLTFComponent } from './GLTFComponent' import { GLTFDocumentState, GLTFModifiedState, GLTFNodeState, GLTFSnapshotAction } from './GLTFDocumentState' -import { GLTFExtensions } from './GLTFExtensions' +import { KHR_DRACO_MESH_COMPRESSION } from './GLTFExtensions' import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' import { MaterialDefinitionComponent } from './MaterialDefinitionComponent' import './MeshExtensionComponents' @@ -807,7 +807,7 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] - const primitive = mesh.primitives[props.primitiveIndex] + const primitive = mesh.primitives[props.primitiveIndex] as GLTF.IMeshPrimitive const geometry = useHookstate(null as null | BufferGeometry) @@ -815,13 +815,8 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do useEffect(() => { if (hasDracoCompression) { - /** @todo */ const options = getParserOptions(props.entity) - GLTFExtensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] - .decodePrimitive(options, documentState.get(NO_PROXY) as GLTF.IGLTF, primitive as GLTF.IMeshPrimitive) - .then((geom) => { - geometry.set(geom) - }) + KHR_DRACO_MESH_COMPRESSION.decodePrimitive(options, primitive).then((geom) => geometry.set(geom)) } else { geometry.set(new BufferGeometry()) } @@ -859,6 +854,14 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do return ( <> + {typeof primitive.extensions === 'object' && ( + + )} {typeof primitive.material === 'number' && ( { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const primitive = mesh.primitives[props.primitiveIndex] + + const extensions = primitive.extensions + + useEffect(() => { + if (!extensions) return + + for (const extension in extensions) { + const Component = ComponentJSONIDMap.get(extension) + if (!Component) continue + setComponent(props.entity, Component, extensions[extension]) + } + }, [extensions]) + + return null +} + const PrimitiveAttributeReactor = (props: { geometry: BufferGeometry attribute: string From 0df162ff8432458deb58aad34cd0c6edb2039262 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 17:45:05 +1000 Subject: [PATCH 21/47] clean up --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 0c6941e8b8..9db7c45c25 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -50,6 +50,7 @@ import { Sphere, Texture, TextureLoader, + Vector2, Vector3 } from 'three' import { FileLoader } from '../assets/loaders/base/FileLoader' @@ -69,6 +70,7 @@ const cache = new GLTFRegistry() const useLoadAccessor = (options: GLTFParserOptions, accessorIndex?: number) => { const json = options.document + const result = useHookstate(null) const accessorDef = typeof accessorIndex === 'number' ? json.accessors![accessorIndex] : null @@ -94,11 +96,9 @@ const useLoadAccessor = (options: GLTFParserOptions, accessorIndex?: number) => return } - if (accessorDef.bufferView && !bufferView) return + if (typeof accessorDef.bufferView === 'number' && !bufferView) return if (accessorDef.sparse && !sparseBufferViewIndices && !sparseBufferViewValues) return - const bufferViews = accessorDef.bufferView ? [bufferView, null] : [sparseBufferViewIndices, sparseBufferViewValues] - const itemSize = WEBGL_TYPE_SIZES[accessorDef.type] const TypedArray = WEBGL_COMPONENT_TYPES[accessorDef.componentType] @@ -161,11 +161,15 @@ const useLoadAccessor = (options: GLTFParserOptions, accessorIndex?: number) => const byteOffsetValues = accessorDef.sparse.values.byteOffset || 0 const sparseIndices = new TypedArrayIndices( - bufferViews[1]!, + sparseBufferViewIndices!, byteOffsetIndices, accessorDef.sparse.count * itemSizeIndices ) - const sparseValues = new TypedArray(bufferViews[2]!, byteOffsetValues, accessorDef.sparse.count * itemSize) + const sparseValues = new TypedArray( + sparseBufferViewValues!, + byteOffsetValues, + accessorDef.sparse.count * itemSize + ) if (bufferView !== null) { // Avoid modifying the original ArrayBuffer, if the bufferView wasn't initialized with zeroes. @@ -472,15 +476,15 @@ const useLoadMaterial = ( if (material) material.needsUpdate = true }, [material, normalMap]) - // useEffect(() => { - // materialParams.normalScale = new Vector2(1, 1) - - // if (materialDef.normalTexture.scale !== undefined) { - // const scale = materialDef.normalTexture.scale - - // materialParams.normalScale.set(scale, scale) - // } - // }, [material, ]) + useEffect(() => { + if (materialDef.normalTexture?.scale) { + const scale = materialDef.normalTexture.scale + result.value?.setValues({ normalScale: new Vector2(scale, scale) }) + } else { + result.value?.setValues({ normalScale: new Vector2(1, 1) }) + } + if (material) material.needsUpdate = true + }, [material, materialDef.normalTexture?.scale]) const aoMap = GLTFLoaderFunctions.useAssignTexture( options, From 8c86f05562dcff548cdc1dfffa94766f284d918f Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 17:58:36 +1000 Subject: [PATCH 22/47] fix quantized accessor and support KHR_TEXTURE_TRANSFORM --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 37 ++++--- .../src/gltf/MaterialDefinitionComponent.tsx | 101 +++++++++++++++++- 2 files changed, 118 insertions(+), 20 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 9db7c45c25..7963fd7127 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -63,7 +63,7 @@ import { } from '../assets/loaders/gltf/GLTFConstants' import { assignExtrasToUserData, getNormalizedComponentScale } from '../assets/loaders/gltf/GLTFLoaderFunctions' import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' -import { MaterialDefinitionComponent } from './MaterialDefinitionComponent' +import { KHRTextureTransformExtensionComponent, MaterialDefinitionComponent } from './MaterialDefinitionComponent' // todo make this a state const cache = new GLTFRegistry() @@ -194,7 +194,7 @@ const useLoadAccessor = (options: GLTFParserOptions, accessorIndex?: number) => result.set(bufferAttribute) }, [bufferView, sparseBufferViewIndices, sparseBufferViewValues]) - return result.value + return result.get(NO_PROXY) } const useLoadBufferView = (options: GLTFParserOptions, bufferViewIndex?: number) => { @@ -213,7 +213,7 @@ const useLoadBufferView = (options: GLTFParserOptions, bufferViewIndex?: number) result.set(buffer.slice(byteOffset, byteOffset + byteLength)) }, [buffer]) - return result.value + return result.get(NO_PROXY) } const useLoadBuffer = (options: GLTFParserOptions, bufferIndex?: number) => { @@ -259,7 +259,7 @@ const useLoadBuffer = (options: GLTFParserOptions, bufferIndex?: number) => { ) }, [bufferDef?.uri]) - return result.value + return result.get(NO_PROXY) } export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primitiveDef: GLTF.IMeshPrimitive) { @@ -524,7 +524,7 @@ const useLoadMaterial = ( if (material) material.needsUpdate = true }, [material, emissiveMap]) - return result.value as MeshStandardMaterial | null + return result.get(NO_PROXY) as MeshStandardMaterial | null } /** @@ -553,21 +553,20 @@ const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo result.set(textureClone) } - result.set(texture) - - // if (GLTFExtensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM]) { - // const transform = - // mapDef.extensions !== undefined ? mapDef.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM] : undefined + const transform = + mapDef.extensions !== undefined ? mapDef.extensions[KHRTextureTransformExtensionComponent.jsonID] : undefined - // if (transform) { - // const gltfReference = parser.associations.get(texture) - // texture = parser.extensions[EXTENSIONS.KHR_TEXTURE_TRANSFORM].extendTexture(texture, transform) - // parser.associations.set(texture, gltfReference) - // } - // } + if (transform) { + // const gltfReference = parser.associations.get(texture) + const extendedTexture = KHRTextureTransformExtensionComponent.extendTexture(texture, transform) + // parser.associations.set(texture, gltfReference) + result.set(extendedTexture) + } else { + result.set(texture) + } }, [texture, mapDef]) - return result.value as Texture | null + return result.get(NO_PROXY) as Texture | null } /** @@ -671,7 +670,7 @@ const useLoadTextureImage = ( // textureCache[cacheKey] = promise - return result.value as Texture | null + return result.get(NO_PROXY) as Texture | null } // const sourceCache = {} as any // todo @@ -765,7 +764,7 @@ const useLoadImageSource = ( // sourceCache[sourceIndex] = promise }, [result.value]) - return result.value as Texture | null + return result.get(NO_PROXY) as Texture | null } export const GLTFLoaderFunctions = { diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx index 0f211147c8..ae7f5f3110 100644 --- a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -36,7 +36,15 @@ import { NO_PROXY } from '@etherealengine/hyperflux' import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTF } from '@gltf-transform/core' import { useEffect, useLayoutEffect } from 'react' -import { Color, LinearSRGBColorSpace, MeshPhysicalMaterial, MeshStandardMaterial, SRGBColorSpace, Vector2 } from 'three' +import { + Color, + LinearSRGBColorSpace, + MeshPhysicalMaterial, + MeshStandardMaterial, + SRGBColorSpace, + Texture, + Vector2 +} from 'three' import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' import { getParserOptions } from './GLTFState' @@ -835,3 +843,94 @@ export const KHRAnisotropyExtensionComponent = defineComponent({ return null } }) + +/** + * Texture Transform Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Khronos/KHR_texture_transform + */ +export class GLTFTextureTransformExtension { + name = EXTENSIONS.KHR_TEXTURE_TRANSFORM + + extendTexture(texture, transform) { + if ( + (transform.texCoord === undefined || transform.texCoord === texture.channel) && + transform.offset === undefined && + transform.rotation === undefined && + transform.scale === undefined + ) { + // See https://github.com/mrdoob/three.js/issues/21819. + return texture + } + + texture = texture.clone() + + if (transform.texCoord !== undefined) { + texture.channel = transform.texCoord + } + + if (transform.offset !== undefined) { + texture.offset.fromArray(transform.offset) + } + + if (transform.rotation !== undefined) { + texture.rotation = transform.rotation + } + + if (transform.scale !== undefined) { + texture.repeat.fromArray(transform.scale) + } + + texture.needsUpdate = true + + return texture + } +} + +type GLTFTextureTransformExtensionType = { + texCoord?: number + offset?: [number, number] + rotation?: number + scale?: [number, number] +} + +export const KHRTextureTransformExtensionComponent = defineComponent({ + name: 'KHRTextureTransformExtensionComponent', + jsonID: EXTENSIONS.KHR_TEXTURE_TRANSFORM, + + /** static function */ + extendTexture: (texture: Texture, transform: GLTFTextureTransformExtensionType) => { + if ( + (transform.texCoord === undefined || transform.texCoord === texture.channel) && + transform.offset === undefined && + transform.rotation === undefined && + transform.scale === undefined + ) { + // See https://github.com/mrdoob/three.js/issues/21819. + return texture + } + + /** @todo this throws hookstate 109... */ + // texture = texture.clone() + + if (transform.texCoord !== undefined) { + texture.channel = transform.texCoord + } + + if (transform.offset !== undefined) { + texture.offset.fromArray(transform.offset) + } + + if (transform.rotation !== undefined) { + texture.rotation = transform.rotation + } + + if (transform.scale !== undefined) { + texture.repeat.fromArray(transform.scale) + } + + texture.needsUpdate = true + + return texture + } +}) From f074919a2e80058c14b5b3f72b2f5a4f136ffae6 Mon Sep 17 00:00:00 2001 From: HexaField Date: Sun, 4 Aug 2024 19:34:04 +1000 Subject: [PATCH 23/47] add KHR_texture_basisu support --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 83 ++++++++++--------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 7963fd7127..1eb2054f0a 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -24,7 +24,7 @@ Ethereal Engine. All Rights Reserved. */ import { ComponentType } from '@etherealengine/ecs' -import { NO_PROXY, useHookstate } from '@etherealengine/hyperflux' +import { NO_PROXY, getState, useHookstate } from '@etherealengine/hyperflux' import { GLTF } from '@gltf-transform/core' import { useEffect } from 'react' import { @@ -41,6 +41,7 @@ import { LinearFilter, LinearMipmapLinearFilter, LinearSRGBColorSpace, + Loader, LoaderUtils, MeshBasicMaterial, MeshPhysicalMaterial, @@ -61,8 +62,11 @@ import { WEBGL_TYPE_SIZES, WEBGL_WRAPPINGS } from '../assets/loaders/gltf/GLTFConstants' +import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' import { assignExtrasToUserData, getNormalizedComponentScale } from '../assets/loaders/gltf/GLTFLoaderFunctions' import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' +import { KTX2Loader } from '../assets/loaders/gltf/KTX2Loader' +import { AssetLoaderState } from '../assets/state/AssetLoaderState' import { KHRTextureTransformExtensionComponent, MaterialDefinitionComponent } from './MaterialDefinitionComponent' // todo make this a state @@ -569,6 +573,29 @@ const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo return result.get(NO_PROXY) as Texture | null } +let isSafari = false +let isFirefox = false +let firefoxVersion = -1 as any // ??? + +if (typeof navigator !== 'undefined') { + isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent) === true + isFirefox = navigator.userAgent.indexOf('Firefox') > -1 + firefoxVersion = isFirefox ? navigator.userAgent.match(/Firefox\/([0-9]+)\./)![1] : -1 +} + +let textureLoader: TextureLoader | ImageBitmapLoader + +/** @todo use resource loader hooks */ +if (typeof createImageBitmap === 'undefined' || isSafari || (isFirefox && firefoxVersion < 98)) { + textureLoader = new TextureLoader() +} else { + textureLoader = new ImageBitmapLoader() +} + +type KHRTextureBasisu = { + source: number +} + /** * Specification: https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#textures * @param {number} textureIndex @@ -578,42 +605,22 @@ const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { const json = options.document const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null - const sourceIndex = textureDef?.source! - const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null - const textureLoader = useHookstate(() => { - let isSafari = false - let isFirefox = false - let firefoxVersion = -1 as any // ??? + const extensions = textureDef?.extensions + const basisu = extensions && (extensions[EXTENSIONS.KHR_TEXTURE_BASISU] as KHRTextureBasisu) - if (typeof navigator !== 'undefined') { - isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent) === true - isFirefox = navigator.userAgent.indexOf('Firefox') > -1 - firefoxVersion = isFirefox ? navigator.userAgent.match(/Firefox\/([0-9]+)\./)![1] : -1 - } - - let textureLoader - - /** @todo use resource loader hooks */ - if (typeof createImageBitmap === 'undefined' || isSafari || (isFirefox && firefoxVersion < 98)) { - textureLoader = new TextureLoader(options.manager) - } else { - textureLoader = new ImageBitmapLoader(options.manager) - } - - return textureLoader - }) - - /** @todo clean all this up */ - - textureLoader.value.setCrossOrigin(options.crossOrigin) - textureLoader.value.setRequestHeader(options.requestHeader) + const sourceIndex = basisu ? basisu.source : textureDef?.source! + const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null - let loader = textureLoader.value + const handler = typeof sourceDef?.uri === 'string' && options.manager.getHandler(sourceDef.uri) + let loader: ImageLoader | ImageBitmapLoader | TextureLoader | KTX2Loader | Loader - if (sourceDef?.uri) { - const handler = options.manager.getHandler(sourceDef.uri) - if (handler !== null) loader = handler + if (handler) loader = handler + if (basisu) loader = getState(AssetLoaderState).gltfLoader.ktx2Loader! + else { + loader = textureLoader + loader.setCrossOrigin(options.crossOrigin) + loader.setRequestHeader(options.requestHeader) } const texture = GLTFLoaderFunctions.useLoadTextureImage(options, textureIndex, sourceIndex, loader) @@ -625,7 +632,7 @@ const useLoadTextureImage = ( options: GLTFParserOptions, textureIndex?: number, sourceIndex?: number, - loader?: ImageLoader | ImageBitmapLoader + loader?: ImageLoader | ImageBitmapLoader | TextureLoader | KTX2Loader | Loader ) => { const json = options.document const result = useHookstate(null) @@ -680,7 +687,7 @@ const URL = self.URL || self.webkitURL const useLoadImageSource = ( options: GLTFParserOptions, sourceIndex?: number, - loader?: ImageLoader | ImageBitmapLoader + loader?: ImageLoader | ImageBitmapLoader | TextureLoader | KTX2Loader | Loader ) => { const json = options.document const result = useHookstate(null) @@ -729,13 +736,13 @@ const useLoadImageSource = ( if (!sourceURI.value) return loader!.load( LoaderUtils.resolveURL(sourceURI.value as string, options.path), - (imageBitmap) => { + (imageBitmap: Texture | ImageBitmap) => { if ((loader as ImageBitmapLoader).isImageBitmapLoader === true) { - const texture = new Texture(imageBitmap) + const texture = new Texture(imageBitmap as ImageBitmap) texture.needsUpdate = true result.set(texture) } else { - result.set(imageBitmap) + result.set(imageBitmap as Texture) } }, undefined, From 128703796265d5819b0cf820ad824843c80240f8 Mon Sep 17 00:00:00 2001 From: HexaField Date: Tue, 6 Aug 2024 22:02:02 +1000 Subject: [PATCH 24/47] support mesh opt extension, prototype better API for extensions --- .../src/assets/loaders/gltf/GLTFLoader.ts | 3 +- .../assets/loaders/gltf/meshopt_decoder.d.ts | 10 ++ packages/engine/src/gltf/GLTFExtensions.ts | 111 ++++++++++++++++++ .../engine/src/gltf/GLTFLoaderFunctions.ts | 29 ++--- 4 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts diff --git a/packages/engine/src/assets/loaders/gltf/GLTFLoader.ts b/packages/engine/src/assets/loaders/gltf/GLTFLoader.ts index 41d9da87c6..97c9da1496 100755 --- a/packages/engine/src/assets/loaders/gltf/GLTFLoader.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFLoader.ts @@ -69,11 +69,12 @@ import { } from './GLTFExtensions' import { GLTFParser } from './GLTFParser' import { KTX2Loader } from './KTX2Loader' +import { MeshoptDecoder } from './meshopt_decoder' export class GLTFLoader extends Loader { dracoLoader = null as null | DRACOLoader ktx2Loader = null as null | KTX2Loader - meshoptDecoder = null + meshoptDecoder = null as null | MeshoptDecoder pluginCallbacks = [] as any[] diff --git a/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts b/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts new file mode 100644 index 0000000000..a3402e37c3 --- /dev/null +++ b/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts @@ -0,0 +1,10 @@ +export type MeshoptDecoder = { + supported: boolean; + ready: Promise; + useWorkers: (numWorkers: number) => void; + decodeVertexBuffer: (buffer: ArrayBuffer, count: number, stride: number, filter: number) => ArrayBuffer; + decodeIndexBuffer: (buffer: ArrayBuffer, count: number, filter: number) => ArrayBuffer; + decodeIndexSequence: (buffer: ArrayBuffer, count: number, filter: number) => ArrayBuffer; + decodeGltfBuffer: (buffer: Uint8Array, count: number, stride: number, source: Uint8Array, mode: number, filter: number) => void; + decodeGltfBufferAsync?: (count: number, stride: number, source: Uint8Array, mode: number, filter: number) => Promise<{ buffer: ArrayBuffer }>; +} \ No newline at end of file diff --git a/packages/engine/src/gltf/GLTFExtensions.ts b/packages/engine/src/gltf/GLTFExtensions.ts index 88f65111b0..2bce8a8767 100644 --- a/packages/engine/src/gltf/GLTFExtensions.ts +++ b/packages/engine/src/gltf/GLTFExtensions.ts @@ -91,3 +91,114 @@ export const KHR_DRACO_MESH_COMPRESSION = { }) } } + +export type KHRMeshOptExtensionType = { + buffer: number + byteOffset?: number + byteLength?: number + byteStride?: number + count: number + mode?: number + filter?: number +} + +/** + * meshopt BufferView Compression Extension + * + * Specification: https://github.com/KhronosGroup/glTF/tree/master/extensions/2.0/Vendor/EXT_meshopt_compression + */ +export const EXT_MESHOPT_COMPRESSION = { + loadBuffer: (options: GLTFParserOptions, bufferViewIndex: number) => { + const json = options.document + const bufferViewDef = json.bufferViews![bufferViewIndex] + const extensionDef = bufferViewDef.extensions![EXTENSIONS.EXT_MESHOPT_COMPRESSION] as KHRMeshOptExtensionType + return [ + extensionDef.buffer, + (bufferView: ArrayBuffer) => + new Promise((resolve, reject) => { + console.log('bufferView', bufferView) + const json = options.document + + const byteOffset = extensionDef.byteOffset || 0 + const byteLength = extensionDef.byteLength || 0 + + const count = extensionDef.count + const stride = extensionDef.byteStride! + + const source = new Uint8Array(bufferView, byteOffset, byteLength) + + const decoder = getState(AssetLoaderState).gltfLoader.meshoptDecoder + if (!decoder || !decoder.supported) { + if (json.extensionsRequired && json.extensionsRequired.indexOf(EXTENSIONS.EXT_MESHOPT_COMPRESSION) >= 0) { + return reject('THREE.GLTFLoader: setMeshoptDecoder must be called before loading compressed files') + } else { + // Assumes that the extension is optional and that fallback buffer data is present + return resolve(null) + } + } + + if (decoder.decodeGltfBufferAsync) { + decoder + .decodeGltfBufferAsync(count, stride, source, extensionDef.mode!, extensionDef.filter!) + .then(function (res) { + resolve(res.buffer) + }) + } else { + // Support for MeshoptDecoder 0.18 or earlier, without decodeGltfBufferAsync + decoder.ready!.then(function () { + const result = new ArrayBuffer(count * stride) + decoder.decodeGltfBuffer( + new Uint8Array(result), + count, + stride, + source, + extensionDef.mode!, + extensionDef.filter! + ) + resolve(result) + }) + } + }) + ] as [number | null, (bufferView: ArrayBuffer) => Promise] + } +} + +type GLTFExtensionType = { + decodePrimitive?: ( + options: GLTFParserOptions, + primitive: GLTF.IMeshPrimitive + ) => Promise> + loadBuffer?: ( + options: GLTFParserOptions, + index: number + ) => [number | null, (bufferView: ArrayBuffer) => Promise] +} + +export const getBufferIndex = (options: GLTFParserOptions, bufferViewIndex?: number) => { + const json = options.document + if (typeof bufferViewIndex !== 'number') + return [null, async (buffer: ArrayBuffer) => buffer] as [ + number | null, + (bufferView: ArrayBuffer) => Promise + ] + const bufferViewDef = json.bufferViews![bufferViewIndex] + for (const extensionName in bufferViewDef.extensions) { + const extension = GLTFExtensions[extensionName] + if (extension.loadBuffer) { + return extension.loadBuffer(options, bufferViewIndex!) + } + } + return [ + bufferViewDef.buffer, + async (buffer) => { + const byteLength = bufferViewDef!.byteLength || 0 + const byteOffset = bufferViewDef!.byteOffset || 0 + return buffer.slice(byteOffset, byteOffset + byteLength) + } + ] as [number | null, (bufferView: ArrayBuffer) => Promise] +} + +export const GLTFExtensions = { + [EXTENSIONS.KHR_DRACO_MESH_COMPRESSION]: KHR_DRACO_MESH_COMPRESSION, + [EXTENSIONS.EXT_MESHOPT_COMPRESSION]: EXT_MESHOPT_COMPRESSION +} as Record diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 1eb2054f0a..136f29975a 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -67,6 +67,7 @@ import { assignExtrasToUserData, getNormalizedComponentScale } from '../assets/l import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' import { KTX2Loader } from '../assets/loaders/gltf/KTX2Loader' import { AssetLoaderState } from '../assets/state/AssetLoaderState' +import { getBufferIndex } from './GLTFExtensions' import { KHRTextureTransformExtensionComponent, MaterialDefinitionComponent } from './MaterialDefinitionComponent' // todo make this a state @@ -202,25 +203,21 @@ const useLoadAccessor = (options: GLTFParserOptions, accessorIndex?: number) => } const useLoadBufferView = (options: GLTFParserOptions, bufferViewIndex?: number) => { - const json = options.document const result = useHookstate(null) - const bufferViewDef = typeof bufferViewIndex === 'number' ? json.bufferViews![bufferViewIndex] : null - const buffer = GLTFLoaderFunctions.useLoadBuffer(options, bufferViewDef?.buffer) - - useEffect(() => { - if (!bufferViewDef || !buffer) return + const [bufferIndex, callback] = getBufferIndex(options, bufferViewIndex) - const byteLength = bufferViewDef.byteLength || 0 - const byteOffset = bufferViewDef.byteOffset || 0 + const buffer = GLTFLoaderFunctions.useLoadBuffer(options, bufferIndex) - result.set(buffer.slice(byteOffset, byteOffset + byteLength)) + useEffect(() => { + if (!buffer) return result.set(null) + callback(buffer).then((buffer) => result.set(buffer)) }, [buffer]) - return result.get(NO_PROXY) + return result.get(NO_PROXY) as ArrayBuffer | null } -const useLoadBuffer = (options: GLTFParserOptions, bufferIndex?: number) => { +const useLoadBuffer = (options: GLTFParserOptions, bufferIndex) => { const json = options.document const result = useHookstate(null) @@ -263,7 +260,7 @@ const useLoadBuffer = (options: GLTFParserOptions, bufferIndex?: number) => { ) }, [bufferDef?.uri]) - return result.get(NO_PROXY) + return result.get(NO_PROXY) as ArrayBuffer | null } export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primitiveDef: GLTF.IMeshPrimitive) { @@ -557,6 +554,7 @@ const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo result.set(textureClone) } + /** @todo properly support extensions */ const transform = mapDef.extensions !== undefined ? mapDef.extensions[KHRTextureTransformExtensionComponent.jsonID] : undefined @@ -606,10 +604,13 @@ const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { const textureDef = typeof textureIndex === 'number' ? json.textures![textureIndex] : null - const extensions = textureDef?.extensions + const extensions = textureDef?.extensions as Record> | null const basisu = extensions && (extensions[EXTENSIONS.KHR_TEXTURE_BASISU] as KHRTextureBasisu) - const sourceIndex = basisu ? basisu.source : textureDef?.source! + /** @todo properly support texture extensions, this is a hack */ + const sourceIndex = + (extensions && Object.values(extensions).find((ext) => typeof ext.source === 'number')?.source) || + textureDef?.source! const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null const handler = typeof sourceDef?.uri === 'string' && options.manager.getHandler(sourceDef.uri) From ac7d471d7dc52c4fb0fb7f447aa10dca28a149c2 Mon Sep 17 00:00:00 2001 From: HexaField Date: Tue, 6 Aug 2024 22:02:35 +1000 Subject: [PATCH 25/47] license --- .../assets/loaders/gltf/meshopt_decoder.d.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts b/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts index a3402e37c3..7c07b9b33c 100644 --- a/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts +++ b/packages/engine/src/assets/loaders/gltf/meshopt_decoder.d.ts @@ -1,3 +1,29 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + + export type MeshoptDecoder = { supported: boolean; ready: Promise; From 9926fc5e60665dc191907ef8a3e915a2c09faf16 Mon Sep 17 00:00:00 2001 From: HexaField Date: Wed, 7 Aug 2024 11:55:13 +1000 Subject: [PATCH 26/47] add EXT_mesh_gpu_instancing component --- .../src/gltf/MeshExtensionComponents.ts | 143 --------- .../src/gltf/MeshExtensionComponents.tsx | 287 ++++++++++++++++++ 2 files changed, 287 insertions(+), 143 deletions(-) delete mode 100644 packages/engine/src/gltf/MeshExtensionComponents.ts create mode 100644 packages/engine/src/gltf/MeshExtensionComponents.tsx diff --git a/packages/engine/src/gltf/MeshExtensionComponents.ts b/packages/engine/src/gltf/MeshExtensionComponents.ts deleted file mode 100644 index 16113b1cd3..0000000000 --- a/packages/engine/src/gltf/MeshExtensionComponents.ts +++ /dev/null @@ -1,143 +0,0 @@ -/* -CPAL-1.0 License - -The contents of this file are subject to the Common Public Attribution License -Version 1.0. (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at -https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. -The License is based on the Mozilla Public License Version 1.1, but Sections 14 -and 15 have been added to cover use of software over a computer network and -provide for limited attribution for the Original Developer. In addition, -Exhibit A has been modified to be consistent with Exhibit B. - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the -specific language governing rights and limitations under the License. - -The Original Code is Ethereal Engine. - -The Original Developer is the Initial Developer. The Initial Developer of the -Original Code is the Ethereal Engine team. - -All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 -Ethereal Engine. All Rights Reserved. -*/ - -import { defineComponent, removeComponent, setComponent, useComponent, useEntityContext } from '@etherealengine/ecs' -import { DirectionalLightComponent, PointLightComponent, SpotLightComponent } from '@etherealengine/spatial' -import { useEffect } from 'react' -import { Color } from 'three' -import { getParserOptions } from './GLTFState' - -export type KHRPunctualLight = { - color?: [number, number, number] - intensity?: number - range?: number - type: 'directional' | 'point' | 'spot' - spot?: { - innerConeAngle?: number - outerConeAngle?: number - } -} - -export const KHRLightsPunctualComponent = defineComponent({ - name: 'KHRLightsPunctualComponent', - jsonID: 'KHR_lights_punctual', - - onInit(entity) { - return {} as { - light?: number - } - }, - - onSet(entity, component, json) { - if (!json) return - if (typeof json.light === 'number') component.light.set(json.light) - }, - - reactor: () => { - const entity = useEntityContext() - const component = useComponent(entity, KHRLightsPunctualComponent) - - const options = getParserOptions(entity) - const json = options.document - const extensions: { - lights?: KHRPunctualLight[] - } = (json.extensions && json.extensions[KHRLightsPunctualComponent.jsonID]) || {} - const lightDefs = extensions.lights - const lightDef = lightDefs && component.light.value ? lightDefs[component.light.value] : undefined - - useEffect(() => { - return () => { - removeComponent(entity, DirectionalLightComponent) - removeComponent(entity, SpotLightComponent) - removeComponent(entity, PointLightComponent) - } - }, [lightDef?.type]) - - useEffect(() => { - if (!lightDef) return - - if (lightDef.type !== 'directional') return - - const color = lightDef.color - ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) - : undefined - const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined - - setComponent(entity, DirectionalLightComponent, { - color, - intensity - }) - }, [lightDef]) - - useEffect(() => { - if (!lightDef) return - - if (lightDef.type !== 'spot') return - - const color = lightDef.color - ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) - : undefined - - const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined - const range = typeof lightDef.range === 'number' ? lightDef.range : undefined - const innerConeAngle = typeof lightDef.spot?.innerConeAngle === 'number' ? lightDef.spot.innerConeAngle : 0 - const outerConeAngle = - typeof lightDef.spot?.outerConeAngle === 'number' ? lightDef.spot.outerConeAngle : Math.PI / 4.0 - - const penumbra = 1.0 - innerConeAngle / outerConeAngle - const angle = outerConeAngle - - setComponent(entity, SpotLightComponent, { - color, - intensity, - decay: 2, - range, - angle, - penumbra - }) - }, [lightDef]) - - useEffect(() => { - if (!lightDef) return - - if (lightDef.type !== 'point') return - - const color = lightDef.color - ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) - : undefined - const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined - const range = typeof lightDef.range === 'number' ? lightDef.range : undefined - - setComponent(entity, PointLightComponent, { - color, - intensity, - decay: 2, - range - }) - }, [lightDef]) - - return null - } -}) diff --git a/packages/engine/src/gltf/MeshExtensionComponents.tsx b/packages/engine/src/gltf/MeshExtensionComponents.tsx new file mode 100644 index 0000000000..6b62ce3c1e --- /dev/null +++ b/packages/engine/src/gltf/MeshExtensionComponents.tsx @@ -0,0 +1,287 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { + Entity, + defineComponent, + removeComponent, + setComponent, + useComponent, + useEntityContext, + useOptionalComponent +} from '@etherealengine/ecs' +import { NO_PROXY, State, none, useHookstate } from '@etherealengine/hyperflux' +import { DirectionalLightComponent, PointLightComponent, SpotLightComponent } from '@etherealengine/spatial' +import { addObjectToGroup, removeObjectFromGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' +import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' +import React, { useEffect } from 'react' +import { + BufferAttribute, + Color, + InstancedBufferAttribute, + InstancedMesh, + Matrix4, + Mesh, + Object3D, + Quaternion, + Vector3 +} from 'three' +import { InstancingComponent } from '../scene/components/InstancingComponent' +import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' +import { getParserOptions } from './GLTFState' + +export type KHRPunctualLight = { + color?: [number, number, number] + intensity?: number + range?: number + type: 'directional' | 'point' | 'spot' + spot?: { + innerConeAngle?: number + outerConeAngle?: number + } +} + +export const KHRLightsPunctualComponent = defineComponent({ + name: 'KHRLightsPunctualComponent', + jsonID: 'KHR_lights_punctual', + + onInit(entity) { + return {} as { + light?: number + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.light === 'number') component.light.set(json.light) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRLightsPunctualComponent) + + const options = getParserOptions(entity) + const json = options.document + const extensions: { + lights?: KHRPunctualLight[] + } = (json.extensions && json.extensions[KHRLightsPunctualComponent.jsonID]) || {} + const lightDefs = extensions.lights + const lightDef = lightDefs && component.light.value ? lightDefs[component.light.value] : undefined + + useEffect(() => { + return () => { + removeComponent(entity, DirectionalLightComponent) + removeComponent(entity, SpotLightComponent) + removeComponent(entity, PointLightComponent) + } + }, [lightDef?.type]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'directional') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + + setComponent(entity, DirectionalLightComponent, { + color, + intensity + }) + }, [lightDef]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'spot') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + const range = typeof lightDef.range === 'number' ? lightDef.range : undefined + const innerConeAngle = typeof lightDef.spot?.innerConeAngle === 'number' ? lightDef.spot.innerConeAngle : 0 + const outerConeAngle = + typeof lightDef.spot?.outerConeAngle === 'number' ? lightDef.spot.outerConeAngle : Math.PI / 4.0 + + const penumbra = 1.0 - innerConeAngle / outerConeAngle + const angle = outerConeAngle + + setComponent(entity, SpotLightComponent, { + color, + intensity, + decay: 2, + range, + angle, + penumbra + }) + }, [lightDef]) + + useEffect(() => { + if (!lightDef) return + + if (lightDef.type !== 'point') return + + const color = lightDef.color + ? new Color().setRGB(lightDef.color[0], lightDef.color[1], lightDef.color[2]) + : undefined + const intensity = typeof lightDef.intensity === 'number' ? lightDef.intensity : undefined + const range = typeof lightDef.range === 'number' ? lightDef.range : undefined + + setComponent(entity, PointLightComponent, { + color, + intensity, + decay: 2, + range + }) + }, [lightDef]) + + return null + } +}) + +export const EXTMeshGPUInstancingComponent = defineComponent({ + name: 'EXTMeshGPUInstancingComponent', + jsonID: 'EXT_mesh_gpu_instancing', + + onInit(entity) { + return {} as { + attributes: Record + } + }, + + onSet(entity, component, json) { + if (!json) return + if (json.attributes) component.attributes.set(json.attributes) + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, EXTMeshGPUInstancingComponent) + const meshComponent = useOptionalComponent(entity, MeshComponent) + + const attributes = component.attributes.get(NO_PROXY) + + const accessorsState = useHookstate({} as Record) + + useEffect(() => { + if (!meshComponent || !attributes) return + // ensure all accessors are loaded + if (Object.keys(attributes).length === 0 || Object.keys(attributes).length !== accessorsState.keys.length) return + processGPUInstancing(entity, accessorsState.get(NO_PROXY), meshComponent.get(NO_PROXY)! as Mesh) + }, [accessorsState, !!meshComponent]) + + return ( + <> + {attributes && + Object.entries(attributes).map(([attribute, accessorIndex]) => ( + + ))} + + ) + } +}) + +const MeshGPUInstancingAttributeReactor = (props: { + accessorsState: State> + entity: Entity + attribute: string + accessorIndex: number +}) => { + const { accessorsState, entity, accessorIndex } = props + const options = getParserOptions(entity) + const accessor = GLTFLoaderFunctions.useLoadAccessor(options, accessorIndex) + + useEffect(() => { + if (!accessor) return + accessorsState[props.attribute].set(accessor) + return () => { + accessorsState[props.attribute].set(none) + } + }, [accessor]) + + return null +} + +const processGPUInstancing = (entity: Entity, attributes: Record, mesh: Mesh) => { + // get any attribute to get the count + const attribute0 = Object.values(attributes)[0] + const count = attribute0.count // All attribute counts should be same + + // For Working + const m = new Matrix4() + const p = new Vector3() + const q = new Quaternion() + const s = new Vector3(1, 1, 1) + + const instancedMesh = new InstancedMesh(mesh.geometry, mesh.material, count) + for (let i = 0; i < count; i++) { + if (attributes.TRANSLATION) { + p.fromBufferAttribute(attributes.TRANSLATION, i) + } + if (attributes.ROTATION) { + q.fromBufferAttribute(attributes.ROTATION, i) + } + if (attributes.SCALE) { + s.fromBufferAttribute(attributes.SCALE, i) + } + // @TODO: Support _ID and others + instancedMesh.setMatrixAt(i, m.compose(p, q, s)) + } + + for (const attributeName in attributes) { + if (attributeName === '_COLOR_0') { + const attr = attributes[attributeName] + instancedMesh.instanceColor = new InstancedBufferAttribute(attr.array, attr.itemSize, attr.normalized) + } else if (attributeName !== 'TRANSLATION' && attributeName !== 'ROTATION' && attributeName !== 'SCALE') { + mesh.geometry.setAttribute(attributeName, attributes[attributeName]) + } + } + + // Just in case + Object3D.prototype.copy.call(instancedMesh, mesh) + + instancedMesh.frustumCulled = false + instancedMesh.instanceMatrix.needsUpdate = true + + /** @todo we really should tidy this up, and change it such that the mesh component itself handles adding and removing from group */ + removeObjectFromGroup(entity, mesh) + removeComponent(entity, MeshComponent) + setComponent(entity, MeshComponent, instancedMesh) + addObjectToGroup(entity, instancedMesh) + + setComponent(entity, InstancingComponent, { + instanceMatrix: instancedMesh.instanceMatrix + }) +} From 28fd15dc841687bdbb0eb941b12b2fe8a045d777 Mon Sep 17 00:00:00 2001 From: HexaField Date: Wed, 7 Aug 2024 13:51:58 +1000 Subject: [PATCH 27/47] bug fix --- .../src/gltf/MaterialDefinitionComponent.tsx | 70 ++++++++++--------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx index ae7f5f3110..322e96263f 100644 --- a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -32,7 +32,7 @@ import { useComponent, useEntityContext } from '@etherealengine/ecs' -import { NO_PROXY } from '@etherealengine/hyperflux' +import { NO_PROXY, useImmediateEffect } from '@etherealengine/hyperflux' import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTF } from '@gltf-transform/core' import { useEffect, useLayoutEffect } from 'react' @@ -160,7 +160,7 @@ export const KHREmissiveStrengthExtensionComponent = defineComponent({ if (typeof component.emissiveStrength.value !== 'number') return const material = materialStateComponent.material.value as MeshStandardMaterial material.setValues({ emissiveIntensity: component.emissiveStrength.value }) - }, [component.emissiveStrength.value]) + }, [materialStateComponent.material.value.type, component.emissiveStrength.value]) return null } @@ -211,14 +211,14 @@ export const KHRClearcoatExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ clearcoat: component.clearcoatFactor.value }) material.needsUpdate = true - }, [component.clearcoatFactor.value]) + }, [materialStateComponent.material.value.type, component.clearcoatFactor.value]) useEffect(() => { if (!component.clearcoatRoughnessFactor.value) return const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ clearcoatRoughness: component.clearcoatRoughnessFactor.value }) material.needsUpdate = true - }, [component.clearcoatRoughnessFactor.value]) + }, [materialStateComponent.material.value.type, component.clearcoatRoughnessFactor.value]) const options = getParserOptions(entity) const clearcoatMap = GLTFLoaderFunctions.useAssignTexture(options, component.clearcoatTexture.value) @@ -227,7 +227,7 @@ export const KHRClearcoatExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ clearcoatMap }) material.needsUpdate = true - }, [clearcoatMap]) + }, [materialStateComponent.material.value.type, clearcoatMap]) const clearcoatRoughnessMap = GLTFLoaderFunctions.useAssignTexture( options, @@ -238,7 +238,7 @@ export const KHRClearcoatExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ clearcoatRoughnessMap }) material.needsUpdate = true - }, [clearcoatRoughnessMap]) + }, [materialStateComponent.material.value.type, clearcoatRoughnessMap]) const clearcoatNormalMap = GLTFLoaderFunctions.useAssignTexture(options, component.clearcoatNormalTexture.value) @@ -252,7 +252,7 @@ export const KHRClearcoatExtensionComponent = defineComponent({ material.setValues({ clearcoatNormalMap }) material.needsUpdate = true - }, [clearcoatNormalMap]) + }, [materialStateComponent.material.value.type, clearcoatNormalMap]) return null } @@ -316,14 +316,14 @@ export const KHRIridescenceExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ iridescence: component.iridescenceFactor.value }) material.needsUpdate = true - }, [component.iridescenceFactor.value]) + }, [materialStateComponent.material.value.type, component.iridescenceFactor.value]) useEffect(() => { if (!component.iridescenceIor.value) return const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ iridescenceIOR: component.iridescenceIor.value }) material.needsUpdate = true - }, [component.iridescenceIor.value]) + }, [materialStateComponent.material.value.type, component.iridescenceIor.value]) const options = getParserOptions(entity) const iridescenceMap = GLTFLoaderFunctions.useAssignTexture(options, component.iridescenceTexture.value) @@ -332,7 +332,7 @@ export const KHRIridescenceExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ iridescenceMap }) material.needsUpdate = true - }, [iridescenceMap]) + }, [materialStateComponent.material.value.type, iridescenceMap]) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial @@ -343,7 +343,11 @@ export const KHRIridescenceExtensionComponent = defineComponent({ ] }) material.needsUpdate = true - }, [component.iridescenceThicknessMinimum.value, component.iridescenceThicknessMaximum.value]) + }, [ + materialStateComponent.material.value.type, + component.iridescenceThicknessMinimum.value, + component.iridescenceThicknessMaximum.value + ]) const iridescenceThicknessMap = GLTFLoaderFunctions.useAssignTexture( options, @@ -354,7 +358,7 @@ export const KHRIridescenceExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ iridescenceThicknessMap }) material.needsUpdate = true - }, [iridescenceThicknessMap]) + }, [materialStateComponent.material.value.type, iridescenceThicknessMap]) return null } @@ -418,14 +422,14 @@ export const KHRSheenExtensionComponent = defineComponent({ ) }) material.needsUpdate = true - }, [component.sheenColorFactor.value]) + }, [materialStateComponent.material.value.type, component.sheenColorFactor.value]) useEffect(() => { if (!component.sheenRoughnessFactor.value) return const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ sheenRoughness: component.sheenRoughnessFactor.value }) material.needsUpdate = true - }, [component.sheenRoughnessFactor.value]) + }, [materialStateComponent.material.value.type, component.sheenRoughnessFactor.value]) const options = getParserOptions(entity) const sheenColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.sheenColorTexture.value) @@ -435,7 +439,7 @@ export const KHRSheenExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ sheenColorMap }) material.needsUpdate = true - }, [sheenColorMap]) + }, [materialStateComponent.material.value.type, sheenColorMap]) const sheenRoughnessMap = GLTFLoaderFunctions.useAssignTexture(options, component.sheenRoughnessTexture.value) @@ -443,7 +447,7 @@ export const KHRSheenExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ sheenRoughnessMap }) material.needsUpdate = true - }, [sheenRoughnessMap]) + }, [materialStateComponent.material.value.type, sheenRoughnessMap]) return null } @@ -493,7 +497,7 @@ export const KHRTransmissionExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ transmission: component.transmissionFactor.value }) material.needsUpdate = true - }, [component.transmissionFactor.value]) + }, [materialStateComponent.material.value.type, component.transmissionFactor.value]) const options = getParserOptions(entity) const transmissionMap = GLTFLoaderFunctions.useAssignTexture(options, component.transmissionTexture.value) @@ -502,7 +506,7 @@ export const KHRTransmissionExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ transmissionMap }) material.needsUpdate = true - }, [transmissionMap]) + }, [materialStateComponent.material.value.type, transmissionMap]) return null } @@ -556,13 +560,13 @@ export const KHRVolumeExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ thickness: component.thicknessFactor.value ?? 0 }) material.needsUpdate = true - }, [component.thicknessFactor.value]) + }, [materialStateComponent.material.value.type, component.thicknessFactor.value]) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ attenuationDistance: component.attenuationDistance.value || Infinity }) material.needsUpdate = true - }, [component.attenuationDistance.value]) + }, [materialStateComponent.material.value.type, component.attenuationDistance.value]) useEffect(() => { if (!component.attenuationColorFactor.value) return @@ -576,7 +580,7 @@ export const KHRVolumeExtensionComponent = defineComponent({ ) }) material.needsUpdate = true - }, [component.attenuationColorFactor.value]) + }, [materialStateComponent.material.value.type, component.attenuationColorFactor.value]) const options = getParserOptions(entity) const thicknessMap = GLTFLoaderFunctions.useAssignTexture(options, component.thicknessTexture.value) @@ -585,7 +589,7 @@ export const KHRVolumeExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ thicknessMap }) material.needsUpdate = true - }, [thicknessMap]) + }, [materialStateComponent.material.value.type, thicknessMap]) return null } @@ -630,7 +634,7 @@ export const KHRIorExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ ior: component.ior.value ?? 1.5 }) material.needsUpdate = true - }, [component.ior.value]) + }, [materialStateComponent.material.value.type, component.ior.value]) return null } @@ -676,7 +680,7 @@ export const KHRSpecularExtensionComponent = defineComponent({ const component = useComponent(entity, KHRSpecularExtensionComponent) const materialStateComponent = useComponent(entity, MaterialStateComponent) - useEffect(() => { + useImmediateEffect(() => { setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) }, []) @@ -684,7 +688,7 @@ export const KHRSpecularExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ specularIntensity: component.specularFactor.value ?? 1.0 }) material.needsUpdate = true - }, [component.specularFactor.value]) + }, [materialStateComponent.material.value.type, component.specularFactor.value]) useEffect(() => { const specularColorFactor = component.specularColorFactor.value ?? [1, 1, 1] @@ -698,7 +702,7 @@ export const KHRSpecularExtensionComponent = defineComponent({ ) }) material.needsUpdate = true - }, [component.specularColorFactor.value]) + }, [materialStateComponent.material.value.type, component.specularColorFactor.value]) const options = getParserOptions(entity) const specularMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularTexture.value) @@ -707,7 +711,7 @@ export const KHRSpecularExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ specularIntensityMap: specularMap }) material.needsUpdate = true - }, [specularMap]) + }, [materialStateComponent.material.value.type, specularMap]) const specularColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularColorTexture.value) @@ -715,7 +719,7 @@ export const KHRSpecularExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ specularColorMap }) material.needsUpdate = true - }, [specularColorMap]) + }, [materialStateComponent.material.value.type, specularColorMap]) return null } @@ -763,7 +767,7 @@ export const EXTBumpExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ bumpScale: component.bumpFactor.value ?? 1.0 }) material.needsUpdate = true - }, [component.bumpFactor.value]) + }, [materialStateComponent.material.value.type, component.bumpFactor.value]) const options = getParserOptions(entity) const bumpMap = GLTFLoaderFunctions.useAssignTexture(options, component.bumpTexture.value) @@ -772,7 +776,7 @@ export const EXTBumpExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ bumpMap }) material.needsUpdate = true - }, [bumpMap]) + }, [materialStateComponent.material.value.type, bumpMap]) return null } @@ -823,13 +827,13 @@ export const KHRAnisotropyExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ anisotropy: component.anisotropyStrength.value ?? 0.0 }) material.needsUpdate = true - }, [component.anisotropyStrength.value]) + }, [materialStateComponent.material.value.type, component.anisotropyStrength.value]) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ anisotropyRotation: component.anisotropyRotation.value ?? 0.0 }) material.needsUpdate = true - }, [component.anisotropyRotation.value]) + }, [materialStateComponent.material.value.type, component.anisotropyRotation.value]) const options = getParserOptions(entity) const anisotropyMap = GLTFLoaderFunctions.useAssignTexture(options, component.anisotropyTexture.value) @@ -838,7 +842,7 @@ export const KHRAnisotropyExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ anisotropyMap }) material.needsUpdate = true - }, [anisotropyMap]) + }, [materialStateComponent.material.value.type, anisotropyMap]) return null } From e92eda5951092c2b29e43c64f597a105eb69aa89 Mon Sep 17 00:00:00 2001 From: HexaField Date: Fri, 9 Aug 2024 14:28:15 +1000 Subject: [PATCH 28/47] refactor primitive loading, add skins and bones --- .../avatar/components/SkinnedMeshComponent.ts | 57 ++++- .../engine/src/gltf/GLTFLoaderFunctions.ts | 86 ++++++- packages/engine/src/gltf/GLTFState.tsx | 221 ++++++++---------- 3 files changed, 238 insertions(+), 126 deletions(-) diff --git a/packages/engine/src/avatar/components/SkinnedMeshComponent.ts b/packages/engine/src/avatar/components/SkinnedMeshComponent.ts index 2849274c47..f2acaf10fe 100644 --- a/packages/engine/src/avatar/components/SkinnedMeshComponent.ts +++ b/packages/engine/src/avatar/components/SkinnedMeshComponent.ts @@ -23,10 +23,28 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { SkinnedMesh } from 'three' +import { SkeletonHelper, SkinnedMesh } from 'three' -import { defineComponent } from '@etherealengine/ecs/src/ComponentFunctions' +import { createEntity, removeEntity, useEntityContext } from '@etherealengine/ecs' +import { + defineComponent, + setComponent, + useComponent, + useOptionalComponent +} from '@etherealengine/ecs/src/ComponentFunctions' +import { getMutableState, getState, useHookstate } from '@etherealengine/hyperflux' +import { EngineState } from '@etherealengine/spatial/src/EngineState' +import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' +import { RendererState } from '@etherealengine/spatial/src/renderer/RendererState' +import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' +import { setObjectLayers } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' +import { VisibleComponent, setVisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' +import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' +import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' +import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { useEffect } from 'react' +/** @todo move this to spatial module */ export const SkinnedMeshComponent = defineComponent({ name: 'SkinnedMeshComponent', @@ -35,5 +53,40 @@ export const SkinnedMeshComponent = defineComponent({ onSet: (entity, component, mesh: SkinnedMesh) => { if (!mesh || !mesh.isSkinnedMesh) throw new Error('SkinnedMeshComponent: Invalid skinned mesh') component.set(mesh) + }, + + reactor: function () { + const entity = useEntityContext() + const component = useComponent(entity, SkinnedMeshComponent) + const debugEnabled = useHookstate(getMutableState(RendererState).avatarDebug) + const visible = useOptionalComponent(entity, VisibleComponent) + + useEffect(() => { + if (!visible?.value || !debugEnabled.value) return + + const helper = new SkeletonHelper(component.value as SkinnedMesh) + helper.frustumCulled = false + helper.name = `Skinned Mesh Helper For: ${entity}` + + const helperEntity = createEntity() + setVisibleComponent(helperEntity, true) + addObjectToGroup(helperEntity, helper) + setComponent(helperEntity, NameComponent, helper.name) + setObjectLayers(helper, ObjectLayers.AvatarHelper) + setComponent(helperEntity, EntityTreeComponent, { parentEntity: getState(EngineState).originEntity }) + + setComponent(helperEntity, ComputedTransformComponent, { + referenceEntities: [entity], + computeFunction: () => { + // this updates the bone helper lines + helper.updateMatrixWorld(true) + } + }) + return () => { + removeEntity(helperEntity) + } + }, [visible, debugEnabled, component.skeleton.value]) + + return null } }) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 136f29975a..33136911b7 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -24,7 +24,7 @@ Ethereal Engine. All Rights Reserved. */ import { ComponentType } from '@etherealengine/ecs' -import { NO_PROXY, getState, useHookstate } from '@etherealengine/hyperflux' +import { NO_PROXY, getState, startReactor, useHookstate } from '@etherealengine/hyperflux' import { GLTF } from '@gltf-transform/core' import { useEffect } from 'react' import { @@ -32,6 +32,7 @@ import { BufferAttribute, BufferGeometry, Color, + ColorManagement, DoubleSide, FrontSide, ImageBitmapLoader, @@ -57,6 +58,7 @@ import { import { FileLoader } from '../assets/loaders/base/FileLoader' import { ALPHA_MODES, + ATTRIBUTES, WEBGL_COMPONENT_TYPES, WEBGL_FILTERS, WEBGL_TYPE_SIZES, @@ -67,12 +69,91 @@ import { assignExtrasToUserData, getNormalizedComponentScale } from '../assets/l import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' import { KTX2Loader } from '../assets/loaders/gltf/KTX2Loader' import { AssetLoaderState } from '../assets/state/AssetLoaderState' -import { getBufferIndex } from './GLTFExtensions' +import { KHR_DRACO_MESH_COMPRESSION, getBufferIndex } from './GLTFExtensions' import { KHRTextureTransformExtensionComponent, MaterialDefinitionComponent } from './MaterialDefinitionComponent' // todo make this a state const cache = new GLTFRegistry() +const useLoadPrimitive = (options: GLTFParserOptions, nodeIndex: number, primitiveIndex: number) => { + const result = useHookstate(null as null | BufferGeometry) + + const json = options.document + const node = json.nodes![nodeIndex]! + const mesh = json.meshes![node.mesh!] + + const primitive = mesh.primitives[primitiveIndex] + + const hasDracoCompression = primitive.extensions && primitive.extensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] + + useEffect(() => { + if (ColorManagement.workingColorSpace !== LinearSRGBColorSpace && 'COLOR_0' in primitive.attributes) { + console.warn( + `THREE.GLTFLoader: Converting vertex colors from "srgb-linear" to "${ColorManagement.workingColorSpace}" not supported.` + ) + } + + if (hasDracoCompression) { + KHR_DRACO_MESH_COMPRESSION.decodePrimitive(options, primitive).then((geom) => { + GLTFLoaderFunctions.computeBounds(json, geom, primitive) + assignExtrasToUserData(geom, primitive as GLTF.IMeshPrimitive) + result.set(geom) + }) + } else { + const geometry = new BufferGeometry() + + /** @todo we need to figure out a better way of handling reactivity for both draco and regular buffers */ + const reactor = startReactor(() => { + const attributes = primitive.attributes + const resourcesState = useHookstate( + () => + ({ + ...Object.fromEntries(Object.keys(attributes).map((key) => [key, false])), + index: false + }) as Record + ) + + for (const attributeName of Object.keys(attributes)) { + const threeAttributeName = ATTRIBUTES[attributeName] || attributeName.toLowerCase() + const attribute = primitive.attributes[attributeName] + const accessor = GLTFLoaderFunctions.useLoadAccessor(options, attribute) + useEffect(() => { + if (!accessor) return + geometry.setAttribute(threeAttributeName, accessor) + resourcesState[attributeName].set(true) + }, [accessor]) + } + + const accessor = GLTFLoaderFunctions.useLoadAccessor(options, primitive.indices!) + + useEffect(() => { + if (!accessor) return + geometry.setIndex(accessor) + resourcesState.index.set(true) + }, [accessor]) + + useEffect(() => { + const attributeCount = Object.keys(attributes).length + const resourcesLoaded = Object.values(resourcesState.get(NO_PROXY)).filter(Boolean).length + if (resourcesLoaded !== attributeCount + (typeof primitive.indices === 'number' ? 1 : 0)) return + + GLTFLoaderFunctions.computeBounds(json, geometry, primitive) + assignExtrasToUserData(geometry, primitive as GLTF.IMeshPrimitive) + result.set(geometry) + reactor.stop() + }, [resourcesState]) + + return null + }) + return () => { + reactor.stop() + } + } + }, [primitive.extensions]) + + return result.get(NO_PROXY) as BufferGeometry | null +} + const useLoadAccessor = (options: GLTFParserOptions, accessorIndex?: number) => { const json = options.document @@ -777,6 +858,7 @@ const useLoadImageSource = ( export const GLTFLoaderFunctions = { computeBounds, + useLoadPrimitive, useLoadAccessor, useLoadBufferView, useLoadBuffer, diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index a31fd3e4fa..2c0523d33e 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -26,16 +26,16 @@ Ethereal Engine. All Rights Reserved. import { GLTF } from '@gltf-transform/core' import React, { useEffect, useLayoutEffect } from 'react' import { - BufferGeometry, - ColorManagement, + Bone, Group, - LinearSRGBColorSpace, LoaderUtils, MathUtils, Matrix4, Mesh, MeshBasicMaterial, Quaternion, + Skeleton, + SkinnedMesh, Vector3 } from 'three' @@ -54,6 +54,7 @@ import { setComponent, UndefinedEntity, useComponent, + useOptionalComponent, UUIDComponent } from '@etherealengine/ecs' import { @@ -86,16 +87,14 @@ import { EngineState } from '@etherealengine/spatial/src/EngineState' import { Physics } from '@etherealengine/spatial/src/physics/classes/Physics' import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' import { MaterialInstanceComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' -import { ATTRIBUTES } from '../assets/loaders/gltf/GLTFConstants' -import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' -import { assignExtrasToUserData } from '../assets/loaders/gltf/GLTFLoaderFunctions' import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' import { AssetLoaderState } from '../assets/state/AssetLoaderState' +import { BoneComponent } from '../avatar/components/BoneComponent' +import { SkinnedMeshComponent } from '../avatar/components/SkinnedMeshComponent' import { SourceComponent } from '../scene/components/SourceComponent' import { proxifyParentChildRelationships } from '../scene/functions/loadGLTFModel' import { GLTFComponent } from './GLTFComponent' import { GLTFDocumentState, GLTFModifiedState, GLTFNodeState, GLTFSnapshotAction } from './GLTFDocumentState' -import { KHR_DRACO_MESH_COMPRESSION } from './GLTFExtensions' import { GLTFLoaderFunctions } from './GLTFLoaderFunctions' import { MaterialDefinitionComponent } from './MaterialDefinitionComponent' import './MeshExtensionComponents' @@ -567,6 +566,20 @@ const ParentNodeReactor = (props: { return } +const getNodeUUID = (node: GLTF.INode, documentID: string, nodeIndex: number) => + (node.extensions?.[UUIDComponent.jsonID] as EntityUUID) ?? (`${documentID}-${nodeIndex}` as EntityUUID) + +const isBoneNode = (json: GLTF.IGLTF, nodeIndex: number) => { + const skinDefs = json.skins || [] + for (let skinIndex = 0, skinLength = skinDefs.length; skinIndex < skinLength; skinIndex++) { + const joints = skinDefs[skinIndex].joints + for (let i = 0, il = joints.length; i < il; i++) { + if (joints[i] === nodeIndex) return true + } + } + return false +} + const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: EntityUUID; documentID: string }) => { const documentState = useMutableState(GLTFDocumentState)[props.documentID] const nodes = documentState.nodes! @@ -579,9 +592,7 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: const entity = entityState.value useEffect(() => { - const uuid = - (node.extensions.value?.[UUIDComponent.jsonID] as EntityUUID) ?? - ((props.documentID + '-' + props.nodeIndex) as EntityUUID) + const uuid = getNodeUUID(node.get(NO_PROXY) as GLTF.IGLTF, props.documentID, props.nodeIndex) const entity = UUIDComponent.getOrCreateEntityByUUID(uuid) setComponent(entity, UUIDComponent, uuid) @@ -614,7 +625,13 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: } if (!hasComponent(entity, Object3DComponent) && !hasComponent(entity, MeshComponent)) { - const obj3d = new Group() + let obj3d: Group | Bone + if (isBoneNode(documentState.get(NO_PROXY) as GLTF.IGLTF, props.nodeIndex)) { + obj3d = new Bone() + setComponent(entity, BoneComponent, obj3d) + } else { + obj3d = new Group() + } obj3d.entity = entity addObjectToGroup(entity, obj3d) proxifyParentChildRelationships(obj3d) @@ -697,6 +714,9 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: {typeof node.mesh.get(NO_PROXY) === 'number' && ( )} + {typeof node.skin.get(NO_PROXY) === 'number' && ( + + )} {typeof node.camera.get(NO_PROXY) === 'number' && ( )} @@ -775,6 +795,57 @@ const MeshReactor = (props: { nodeIndex: number; documentID: string; entity: Ent ) } +const SkinnedMeshReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const skinnedMeshComponent = useOptionalComponent(props.entity, SkinnedMeshComponent) + + const skin = documentState.skins.get(NO_PROXY)![node.skin!] + + const options = getParserOptions(props.entity) + const inverseBindMatrices = GLTFLoaderFunctions.useLoadAccessor(options, skin.inverseBindMatrices) + + useEffect(() => { + if (!inverseBindMatrices || !skinnedMeshComponent) return + + const jointNodeUUIDs = skin.joints.map((joint) => getNodeUUID(nodes[joint] as GLTF.INode, props.documentID, joint)) + const jointEntities = jointNodeUUIDs.map((uuid) => UUIDComponent.getEntityByUUID(uuid)) + if (jointEntities.includes(UndefinedEntity)) return + + const jointBones = jointEntities.map((entity) => getOptionalComponent(entity, BoneComponent)) + + if (jointBones.includes(undefined)) return + + const bones = [] as Bone[] + const boneInverses = [] as Matrix4[] + + for (let i = 0, il = jointBones.length; i < il; i++) { + const jointNode = jointBones[i] + + if (jointNode) { + bones.push(jointNode) + + const mat = new Matrix4() + + if (inverseBindMatrices !== null) { + mat.fromArray(inverseBindMatrices.array, i * 16) + } + + boneInverses.push(mat) + } else { + // console.warn('THREE.GLTFLoader: Joint "%s" could not be found.', skinDef.joints[i]) + } + } + + const skeleton = new Skeleton(bones, boneInverses) + skinnedMeshComponent.skeleton.set(skeleton) + }, [inverseBindMatrices, !!skinnedMeshComponent]) + + return null +} + const CameraReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) const nodes = documentState.nodes!.get(NO_PROXY)! @@ -809,48 +880,35 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do const primitive = mesh.primitives[props.primitiveIndex] as GLTF.IMeshPrimitive - const geometry = useHookstate(null as null | BufferGeometry) - - const hasDracoCompression = primitive.extensions && primitive.extensions[EXTENSIONS.KHR_DRACO_MESH_COMPRESSION] + const options = getParserOptions(props.entity) + const geometry = GLTFLoaderFunctions.useLoadPrimitive(options, props.nodeIndex, props.primitiveIndex) - useEffect(() => { - if (hasDracoCompression) { - const options = getParserOptions(props.entity) - KHR_DRACO_MESH_COMPRESSION.decodePrimitive(options, primitive).then((geom) => geometry.set(geom)) + useLayoutEffect(() => { + if (!geometry) return + + let mesh: Mesh | SkinnedMesh + if (typeof node.skin !== 'undefined') { + const skinnedMesh = new SkinnedMesh(geometry, new MeshBasicMaterial()) + mesh = skinnedMesh + skinnedMesh.skeleton = new Skeleton() + setComponent(props.entity, MeshComponent, skinnedMesh) + setComponent(props.entity, SkinnedMeshComponent, skinnedMesh) } else { - geometry.set(new BufferGeometry()) - } - - if (ColorManagement.workingColorSpace !== LinearSRGBColorSpace && 'COLOR_0' in primitive.attributes) { - console.warn( - `THREE.GLTFLoader: Converting vertex colors from "srgb-linear" to "${ColorManagement.workingColorSpace}" not supported.` - ) + mesh = new Mesh(geometry, new MeshBasicMaterial()) + setComponent(props.entity, MeshComponent, mesh) } - }, [primitive.extensions]) - - useEffect(() => { - if (!geometry.value) return - const mesh = new Mesh(geometry.value as BufferGeometry, new MeshBasicMaterial()) /** @todo multiple primitive support */ - setComponent(props.entity, MeshComponent, mesh) addObjectToGroup(props.entity, mesh) - assignExtrasToUserData(geometry, primitive as GLTF.IMeshPrimitive) - - GLTFLoaderFunctions.computeBounds( - documentState.get(NO_PROXY) as GLTF.IGLTF, - geometry.value as BufferGeometry, - primitive as GLTF.IMeshPrimitive - ) - return () => { + removeComponent(props.entity, SkinnedMeshComponent) removeComponent(props.entity, MeshComponent) removeObjectFromGroup(props.entity, mesh) } - }, [geometry]) + }, [node.skin, geometry]) - if (!geometry.value) return null + if (!geometry) return null return ( <> @@ -870,27 +928,6 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do entity={props.entity} /> )} - {!hasDracoCompression && - Object.keys(primitive.attributes).map((attribute, index) => ( - - ))} - {!hasDracoCompression && typeof primitive.indices === 'number' && ( - - )} ) } @@ -925,66 +962,6 @@ const PrimitiveExtensionReactor = (props: { return null } -const PrimitiveAttributeReactor = (props: { - geometry: BufferGeometry - attribute: string - primitiveIndex: number - nodeIndex: number - documentID: string - entity: Entity -}) => { - const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) - - const nodes = documentState.nodes!.get(NO_PROXY)! - const node = nodes[props.nodeIndex]! - - const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] - - const primitive = mesh.primitives[props.primitiveIndex] - - const threeAttributeName = ATTRIBUTES[props.attribute] || props.attribute.toLowerCase() - const attributeAlreadyLoaded = threeAttributeName in props.geometry.attributes - - // Skip attributes already provided by e.g. Draco extension. - const attribute = attributeAlreadyLoaded ? undefined : primitive.attributes[props.attribute] - - const accessor = GLTFLoaderFunctions.useLoadAccessor(getParserOptions(props.entity), attribute) - - useEffect(() => { - if (!accessor) return - - props.geometry.setAttribute(threeAttributeName, accessor) - }, [accessor, props.geometry]) - - return null -} - -const PrimitiveIndicesAttributeReactor = (props: { - geometry: BufferGeometry - primitiveIndex: number - nodeIndex: number - documentID: string - entity: Entity -}) => { - const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) - - const nodes = documentState.nodes!.get(NO_PROXY)! - const node = nodes[props.nodeIndex]! - - const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] - - const primitive = mesh.primitives[props.primitiveIndex] - - const accessor = GLTFLoaderFunctions.useLoadAccessor(getParserOptions(props.entity), primitive.indices!) - - useEffect(() => { - if (!accessor) return - props.geometry.setIndex(accessor) - }, [accessor, props.geometry]) - - return null -} - const MaterialInstanceReactor = (props: { nodeIndex: number documentID: string From f1b73078adcf2f04a863c2af67167adba0206fa1 Mon Sep 17 00:00:00 2001 From: HexaField Date: Fri, 9 Aug 2024 20:51:03 +1000 Subject: [PATCH 29/47] add animation support --- .../src/assets/loaders/gltf/GLTFParser.ts | 1 + .../engine/src/gltf/GLTFLoaderFunctions.ts | 267 +++++++++++++++++- packages/engine/src/gltf/GLTFState.tsx | 97 ++++++- 3 files changed, 353 insertions(+), 12 deletions(-) diff --git a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts index 29bbf7832e..04452c538a 100644 --- a/packages/engine/src/assets/loaders/gltf/GLTFParser.ts +++ b/packages/engine/src/assets/loaders/gltf/GLTFParser.ts @@ -125,6 +125,7 @@ declare module '@gltf-transform/core/dist/types/gltf.d.ts' { export type GLTFParserOptions = { body: null | ArrayBuffer + documentID: string document: GLTF.IGLTF crossOrigin: 'anonymous' | string ktx2Loader: KTX2Loader diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 33136911b7..c719960af0 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -23,11 +23,20 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { ComponentType } from '@etherealengine/ecs' +import { + ComponentType, + EntityUUID, + UUIDComponent, + getOptionalComponent, + useOptionalComponent +} from '@etherealengine/ecs' import { NO_PROXY, getState, startReactor, useHookstate } from '@etherealengine/hyperflux' +import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { GLTF } from '@gltf-transform/core' import { useEffect } from 'react' import { + AnimationClip, + Bone, Box3, BufferAttribute, BufferGeometry, @@ -39,36 +48,51 @@ import { ImageLoader, InterleavedBuffer, InterleavedBufferAttribute, + InterpolateLinear, + KeyframeTrack, LinearFilter, LinearMipmapLinearFilter, LinearSRGBColorSpace, Loader, LoaderUtils, + Mesh, MeshBasicMaterial, MeshPhysicalMaterial, MeshStandardMaterial, + NumberKeyframeTrack, + QuaternionKeyframeTrack, RepeatWrapping, SRGBColorSpace, + SkinnedMesh, Sphere, Texture, TextureLoader, Vector2, - Vector3 + Vector3, + VectorKeyframeTrack } from 'three' import { FileLoader } from '../assets/loaders/base/FileLoader' import { ALPHA_MODES, ATTRIBUTES, + INTERPOLATION, + PATH_PROPERTIES, WEBGL_COMPONENT_TYPES, WEBGL_FILTERS, WEBGL_TYPE_SIZES, WEBGL_WRAPPINGS } from '../assets/loaders/gltf/GLTFConstants' import { EXTENSIONS } from '../assets/loaders/gltf/GLTFExtensions' -import { assignExtrasToUserData, getNormalizedComponentScale } from '../assets/loaders/gltf/GLTFLoaderFunctions' +import { + GLTFCubicSplineInterpolant, + GLTFCubicSplineQuaternionInterpolant, + assignExtrasToUserData, + getNormalizedComponentScale +} from '../assets/loaders/gltf/GLTFLoaderFunctions' import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' import { KTX2Loader } from '../assets/loaders/gltf/KTX2Loader' import { AssetLoaderState } from '../assets/state/AssetLoaderState' +import { BoneComponent } from '../avatar/components/BoneComponent' import { KHR_DRACO_MESH_COMPRESSION, getBufferIndex } from './GLTFExtensions' import { KHRTextureTransformExtensionComponent, MaterialDefinitionComponent } from './MaterialDefinitionComponent' @@ -856,6 +880,240 @@ const useLoadImageSource = ( return result.get(NO_PROXY) as Texture | null } +const getNodeUUID = (node: GLTF.INode, documentID: string, nodeIndex: number) => + (node.extensions?.[UUIDComponent.jsonID] as EntityUUID) ?? (`${documentID}-${nodeIndex}` as EntityUUID) + +const useLoadAnimation = (options: GLTFParserOptions, animationIndex?: number) => { + const result = useHookstate(null as null | AnimationClip) + + const json = options.document + + const animationDef = typeof animationIndex === 'number' ? json.animations![animationIndex] : null + const animationName = animationDef ? (animationDef.name ? animationDef.name : 'animation_' + animationIndex) : null + + useEffect(() => { + if (!animationDef || !animationName) return + + const channels = animationDef.channels.filter((channel) => channel.target.node !== undefined) + + const reactor = startReactor(() => { + const channelData = useHookstate(() => + Object.fromEntries( + channels.map((channel, i) => [ + i, + { + nodes: null as null | Mesh | Bone, + inputAccessors: null as null | BufferAttribute, + outputAccessors: null as null | BufferAttribute, + samplers: animationDef.samplers[channel.sampler], + targets: channel.target + } + ]) + ) + ) + + for (let i = 0, il = channels.length; i < il; i++) { + const channel = channels[i] + const sampler = animationDef.samplers[channel.sampler] + const target = channel.target + const nodeIndex = target.node! + const input = animationDef.parameters !== undefined ? animationDef.parameters[sampler.input] : sampler.input + const output = animationDef.parameters !== undefined ? animationDef.parameters[sampler.output] : sampler.output + + const targetNodeUUID = getNodeUUID(json.nodes![nodeIndex], options.documentID, nodeIndex) + const targetNodeEntity = UUIDComponent.useEntityByUUID(targetNodeUUID) + + /** @todo we should probably jsut use GroupComponent or something here once we stop creating Object3Ds for all nodes */ + const meshComponent = useOptionalComponent(targetNodeEntity, MeshComponent) + const boneComponent = useOptionalComponent(targetNodeEntity, BoneComponent) + + useEffect(() => { + if (!meshComponent && !boneComponent) return + channelData[i].nodes.set( + getOptionalComponent(targetNodeEntity, MeshComponent) ?? + getOptionalComponent(targetNodeEntity, BoneComponent)! + ) + }, [meshComponent, boneComponent]) + + const inputAccessor = GLTFLoaderFunctions.useLoadAccessor(options, input) + + useEffect(() => { + if (!inputAccessor) return + channelData[i].inputAccessors.set(inputAccessor) + }, [inputAccessor]) + + const outputAccessor = GLTFLoaderFunctions.useLoadAccessor(options, output) + + useEffect(() => { + if (!outputAccessor) return + channelData[i].outputAccessors.set(outputAccessor) + }, [outputAccessor]) + } + + useEffect(() => { + if ( + Object.values(channelData.get(NO_PROXY)).some( + (data) => data.nodes === null || data.inputAccessors === null || data.outputAccessors === null + ) + ) + return + + const values = Object.values(channelData.get(NO_PROXY)) + const nodes = values.map((data) => data.nodes) + const inputAccessors = values.map((data) => data.inputAccessors) as BufferAttribute[] + const outputAccessors = values.map((data) => data.outputAccessors) as BufferAttribute[] + const samplers = values.map((data) => data.samplers) as GLTF.IAnimationSampler[] + const targets = values.map((data) => data.targets) as GLTF.IAnimationChannelTarget[] + + const tracks = [] as any[] // todo + + for (let i = 0, il = nodes.length; i < il; i++) { + const node = nodes[i] as Mesh | SkinnedMesh + const inputAccessor = inputAccessors[i] + const outputAccessor = outputAccessors[i] + const sampler = samplers[i] + const target = targets[i] + + if (node === undefined) continue + + if (node.updateMatrix) { + node.updateMatrix() + } + + const createdTracks = _createAnimationTracks(node, inputAccessor, outputAccessor, sampler, target) + + if (createdTracks) { + for (let k = 0; k < createdTracks.length; k++) { + tracks.push(createdTracks[k]) + } + } + } + + result.set(new AnimationClip(animationName, undefined, tracks)) + + reactor.stop() + }, [channelData]) + + return null + }) + return () => { + reactor.stop() + } + }, [animationDef]) + + return result.get(NO_PROXY) as AnimationClip | null +} + +const _createAnimationTracks = ( + node: Mesh | SkinnedMesh, + inputAccessor: BufferAttribute, + outputAccessor: BufferAttribute, + sampler: GLTF.IAnimationSampler, + target: GLTF.IAnimationChannelTarget +) => { + const tracks = [] as any[] // todo + + const targetName = node.name ? node.name : node.uuid + const targetNames = [] as string[] + + if (PATH_PROPERTIES[target.path] === PATH_PROPERTIES.weights) { + node.traverse(function (object: Mesh | SkinnedMesh) { + if (object.morphTargetInfluences) { + targetNames.push(object.name ? object.name : object.uuid) + } + }) + } else { + targetNames.push(targetName) + } + + let TypedKeyframeTrack + + switch (PATH_PROPERTIES[target.path]) { + case PATH_PROPERTIES.weights: + TypedKeyframeTrack = NumberKeyframeTrack + break + + case PATH_PROPERTIES.rotation: + TypedKeyframeTrack = QuaternionKeyframeTrack + break + + case PATH_PROPERTIES.position: + case PATH_PROPERTIES.scale: + TypedKeyframeTrack = VectorKeyframeTrack + break + + default: + switch (outputAccessor.itemSize) { + case 1: + TypedKeyframeTrack = NumberKeyframeTrack + break + case 2: + case 3: + default: + TypedKeyframeTrack = VectorKeyframeTrack + break + } + + break + } + + const interpolation = sampler.interpolation !== undefined ? INTERPOLATION[sampler.interpolation] : InterpolateLinear + + const outputArray = _getArrayFromAccessor(outputAccessor) + + for (let j = 0, jl = targetNames.length; j < jl; j++) { + const track = new TypedKeyframeTrack( + targetNames[j] + '.' + PATH_PROPERTIES[target.path], + inputAccessor.array, + outputArray, + interpolation + ) + + // Override interpolation with custom factory method. + if (sampler.interpolation === 'CUBICSPLINE') { + _createCubicSplineTrackInterpolant(track) + } + + tracks.push(track) + } + + return tracks +} + +const _getArrayFromAccessor = (accessor: BufferAttribute) => { + let outputArray = accessor.array + + if (accessor.normalized) { + const scale = getNormalizedComponentScale(outputArray.constructor) + const scaled = new Float32Array(outputArray.length) + + for (let j = 0, jl = outputArray.length; j < jl; j++) { + scaled[j] = outputArray[j] * scale + } + + outputArray = scaled + } + + return outputArray +} + +const _createCubicSplineTrackInterpolant = (track: KeyframeTrack) => { + // @ts-ignore + track.createInterpolant = function InterpolantFactoryMethodGLTFCubicSpline(result) { + // A CUBICSPLINE keyframe in glTF has three output values for each input value, + // representing inTangent, splineVertex, and outTangent. As a result, track.getValueSize() + // must be divided by three to get the interpolant's sampleSize argument. + + const interpolantType = + this instanceof QuaternionKeyframeTrack ? GLTFCubicSplineQuaternionInterpolant : GLTFCubicSplineInterpolant + + return new interpolantType(this.times, this.values, this.getValueSize() / 3, result) + } + + // @ts-ignore Mark as CUBICSPLINE. `track.getInterpolation()` doesn't support custom interpolants. + track.createInterpolant.isInterpolantFactoryMethodGLTFCubicSpline = true +} + export const GLTFLoaderFunctions = { computeBounds, useLoadPrimitive, @@ -866,5 +1124,6 @@ export const GLTFLoaderFunctions = { useAssignTexture, useLoadTexture, useLoadImageSource, - useLoadTextureImage + useLoadTextureImage, + useLoadAnimation } diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 2c0523d33e..5299c5cdae 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -26,6 +26,8 @@ Ethereal Engine. All Rights Reserved. import { GLTF } from '@gltf-transform/core' import React, { useEffect, useLayoutEffect } from 'react' import { + AnimationClip, + AnimationMixer, Bone, Group, LoaderUtils, @@ -89,6 +91,7 @@ import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/ import { MaterialInstanceComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' import { AssetLoaderState } from '../assets/state/AssetLoaderState' +import { AnimationComponent } from '../avatar/components/AnimationComponent' import { BoneComponent } from '../avatar/components/BoneComponent' import { SkinnedMeshComponent } from '../avatar/components/SkinnedMeshComponent' import { SourceComponent } from '../scene/components/SourceComponent' @@ -413,7 +416,27 @@ const ChildGLTFReactor = (props: { source: string }) => { export const DocumentReactor = (props: { documentID: string; parentUUID: EntityUUID }) => { const nodeState = useHookstate(getMutableState(GLTFNodeState)[props.documentID]) const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const animationState = useHookstate([] as AnimationClip[]) + const rootEntity = UUIDComponent.useEntityByUUID(props.parentUUID) + + useEffect(() => { + if (!documentState.value || !nodeState.value || animationState.length !== documentState.value.animations?.length) + return + + const scene = getComponent(rootEntity, Object3DComponent) + scene.animations = animationState.get(NO_PROXY) as AnimationClip[] + const mixer = new AnimationMixer(scene) + setComponent(rootEntity, AnimationComponent, { + mixer, + animations: scene.animations + }) + return () => { + removeComponent(rootEntity, AnimationComponent) + } + }, [animationState]) + if (!documentState.value || !nodeState.value) return null + return ( <> {Object.entries(nodeState.get(NO_PROXY)).map(([uuid, { nodeIndex, childIndex, parentUUID }]) => ( @@ -435,6 +458,17 @@ export const DocumentReactor = (props: { documentID: string; parentUUID: EntityU documentID={props.documentID} /> ))} + {documentState.get(NO_PROXY).animations?.map((animation, index) => { + return ( + + ) + })} ) } @@ -807,15 +841,21 @@ const SkinnedMeshReactor = (props: { nodeIndex: number; documentID: string; enti const options = getParserOptions(props.entity) const inverseBindMatrices = GLTFLoaderFunctions.useLoadAccessor(options, skin.inverseBindMatrices) + const jointNodeUUIDs = skin.joints.map((joint) => + getNodeUUID(nodes[joint] as GLTF.INode, props.documentID, joint) + ) as EntityUUID[] + /** @todo make reactive to edits */ + const jointEntityLoadedState = useHookstate(() => + Object.fromEntries(jointNodeUUIDs.map((uuid) => [uuid, UndefinedEntity])) + ) + useEffect(() => { if (!inverseBindMatrices || !skinnedMeshComponent) return - const jointNodeUUIDs = skin.joints.map((joint) => getNodeUUID(nodes[joint] as GLTF.INode, props.documentID, joint)) - const jointEntities = jointNodeUUIDs.map((uuid) => UUIDComponent.getEntityByUUID(uuid)) + const jointEntities = Object.values(jointEntityLoadedState.value) if (jointEntities.includes(UndefinedEntity)) return const jointBones = jointEntities.map((entity) => getOptionalComponent(entity, BoneComponent)) - if (jointBones.includes(undefined)) return const bones = [] as Bone[] @@ -841,7 +881,34 @@ const SkinnedMeshReactor = (props: { nodeIndex: number; documentID: string; enti const skeleton = new Skeleton(bones, boneInverses) skinnedMeshComponent.skeleton.set(skeleton) - }, [inverseBindMatrices, !!skinnedMeshComponent]) + }, [jointEntityLoadedState, inverseBindMatrices, !!skinnedMeshComponent]) + + return ( + <> + {jointEntityLoadedState.keys.map((entityUUID: EntityUUID) => ( + + ))} + + ) +} + +/** @todo we can probably simplify this into a nested reactor in a 'useNodesLoaded' entity */ +const SkeletonNodeDependencyReactor = (props: { + entityUUID: EntityUUID + jointEntityLoadedState: State> +}) => { + const entity = UUIDComponent.useEntityByUUID(props.entityUUID) + const jointEntityLoadedState = props.jointEntityLoadedState + + useEffect(() => { + if (!entity) return + jointEntityLoadedState[props.entityUUID].set(entity) + return () => jointEntityLoadedState[props.entityUUID].set(UndefinedEntity) + }, [entity]) return null } @@ -990,10 +1057,23 @@ const MaterialInstanceReactor = (props: { return null } -/** - * TODO figure out how to support extensions that change the behaviour of these reactors - * - we pretty much have to add a new API for each dependency type, like how the GLTFLoader does - */ +export const AnimationReactor = (props: { + index: number + documentID: string + parentUUID: EntityUUID + animationState: State +}) => { + const entity = UUIDComponent.useEntityByUUID(props.parentUUID) + const options = getParserOptions(entity) + const animationTrack = GLTFLoaderFunctions.useLoadAnimation(options, props.index) + + useEffect(() => { + if (!animationTrack) return + props.animationState.merge([animationTrack]) + }, [animationTrack]) + + return null +} export const getParserOptions = (entity: Entity) => { const gltfEntity = getAncestorWithComponent(entity, GLTFComponent) @@ -1002,6 +1082,7 @@ export const getParserOptions = (entity: Entity) => { const gltfLoader = getState(AssetLoaderState).gltfLoader return { document, + documentID: getComponent(gltfEntity, SourceComponent), url: gltfComponent.src, path: LoaderUtils.extractUrlBase(gltfComponent.src), body: gltfComponent.body, From 9e2c611b7cc7efbafc5b07522b19f4a0f235c0ef Mon Sep 17 00:00:00 2001 From: HexaField Date: Mon, 12 Aug 2024 09:30:36 +1000 Subject: [PATCH 30/47] add morph target support --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 63 ++++++++++++++++++- packages/engine/src/gltf/GLTFState.tsx | 58 ++++++++++++++++- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index c719960af0..dc3e93ddd3 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -633,6 +633,60 @@ const useLoadMaterial = ( return result.get(NO_PROXY) as MeshStandardMaterial | null } +const useLoadMorphTargets = (options: GLTFParserOptions, targetsList: Record[]) => { + const result = useHookstate(null as null | Record) + + useEffect(() => { + /** @todo make individual targets individually reactive */ + const reactor = startReactor(() => { + const targetState = useHookstate( + () => + targetsList.map((target) => Object.fromEntries(Object.entries(target).map(([key]) => [key, null]))) as Record< + string, + BufferAttribute | null + >[] + ) + + for (let i = 0, il = targetsList.length; i < il; i++) { + const target = targetsList[i] + for (const [key, accessorIndex] of Object.entries(target)) { + const accessor = GLTFLoaderFunctions.useLoadAccessor(options, accessorIndex) + useEffect(() => { + if (!accessor) return + targetState[i][key].set(accessor) + }, [accessor]) + } + } + + useEffect(() => { + for (const target of targetState.value) { + if (Object.values(target).includes(null)) return + } + result.set( + targetState.get(NO_PROXY).reduce( + (acc, target: Record) => { + for (const [key, value] of Object.entries(target)) { + if (!acc[key]) acc[key] = [] + acc[key].push(value) + } + return acc + }, + {} as Record + ) + ) + reactor.stop() + }, [targetState]) + + return null + }) + return () => { + reactor.stop() + } + }, [targetsList]) + + return result.get(NO_PROXY) as Record | null +} + /** * Asynchronously assigns a texture to the given material parameters. * @param {Object} materialParams @@ -919,6 +973,9 @@ const useLoadAnimation = (options: GLTFParserOptions, animationIndex?: number) = const nodeIndex = target.node! const input = animationDef.parameters !== undefined ? animationDef.parameters[sampler.input] : sampler.input const output = animationDef.parameters !== undefined ? animationDef.parameters[sampler.output] : sampler.output + const node = json.nodes![nodeIndex] + const mesh = typeof node.mesh === 'number' ? json.meshes?.[node.mesh!] : null + const meshHasWeights = mesh?.weights !== undefined && mesh.weights.length > 0 const targetNodeUUID = getNodeUUID(json.nodes![nodeIndex], options.documentID, nodeIndex) const targetNodeEntity = UUIDComponent.useEntityByUUID(targetNodeUUID) @@ -928,7 +985,10 @@ const useLoadAnimation = (options: GLTFParserOptions, animationIndex?: number) = const boneComponent = useOptionalComponent(targetNodeEntity, BoneComponent) useEffect(() => { - if (!meshComponent && !boneComponent) return + const meshWeightsLoaded = meshHasWeights + ? meshComponent?.get(NO_PROXY)?.morphTargetInfluences !== undefined + : true + if (!meshWeightsLoaded && !boneComponent) return channelData[i].nodes.set( getOptionalComponent(targetNodeEntity, MeshComponent) ?? getOptionalComponent(targetNodeEntity, BoneComponent)! @@ -1121,6 +1181,7 @@ export const GLTFLoaderFunctions = { useLoadBufferView, useLoadBuffer, useLoadMaterial, + useLoadMorphTargets, useAssignTexture, useLoadTexture, useLoadImageSource, diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 5299c5cdae..5ae8ccb34c 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -50,6 +50,7 @@ import { getComponent, getMutableComponent, getOptionalComponent, + getOptionalMutableComponent, hasComponent, removeComponent, removeEntity, @@ -943,9 +944,9 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do const nodes = documentState.nodes!.get(NO_PROXY)! const node = nodes[props.nodeIndex]! - const mesh = documentState.meshes.get(NO_PROXY)![node.mesh!] + const meshDef = documentState.meshes.get(NO_PROXY)![node.mesh!] - const primitive = mesh.primitives[props.primitiveIndex] as GLTF.IMeshPrimitive + const primitive = meshDef.primitives[props.primitiveIndex] as GLTF.IMeshPrimitive const options = getParserOptions(props.entity) const geometry = GLTFLoaderFunctions.useLoadPrimitive(options, props.nodeIndex, props.primitiveIndex) @@ -995,6 +996,16 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do entity={props.entity} /> )} + {primitive.targets && ( + + )} ) } @@ -1057,6 +1068,49 @@ const MaterialInstanceReactor = (props: { return null } +export const MorphTargetReactor = (props: { + documentID: string + entity: Entity + nodeIndex: number + primitiveIndex: number + targets: Record[] +}) => { + const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + + const nodes = documentState.nodes!.get(NO_PROXY)! + const node = nodes[props.nodeIndex]! + + const meshDef = documentState.meshes.get(NO_PROXY)![node.mesh!] + + const options = getParserOptions(props.entity) + const morphTargets = GLTFLoaderFunctions.useLoadMorphTargets(options, props.targets) + + useEffect(() => { + if (!morphTargets) return + + const mesh = getOptionalMutableComponent(props.entity, MeshComponent) + if (!mesh) return + + if (morphTargets.POSITION) mesh.geometry.morphAttributes.position.set(morphTargets.POSITION) + if (morphTargets.NORMAL) mesh.geometry.morphAttributes.normal.set(morphTargets.NORMAL) + if (morphTargets.COLOR_0) mesh.geometry.morphAttributes.color.set(morphTargets.COLOR_0) + + mesh.geometry.morphTargetsRelative.set(true) + + mesh.get(NO_PROXY).updateMorphTargets() + + if (meshDef.weights) { + for (let i = 0, il = meshDef.weights.length; i < il; i++) { + mesh.morphTargetInfluences[i].set(meshDef.weights[i]) + } + } + + console.log('Morph targets loaded', mesh) + }, [morphTargets]) + + return null +} + export const AnimationReactor = (props: { index: number documentID: string From 13267936fd20982677a4dec453baa5a2dcff8444 Mon Sep 17 00:00:00 2001 From: HexaField Date: Mon, 12 Aug 2024 11:43:43 +1000 Subject: [PATCH 31/47] bug fixes, add old specular gloss and lightmap extensions --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 4 +- .../src/gltf/MaterialDefinitionComponent.tsx | 175 ++++++++++++++++++ 2 files changed, 176 insertions(+), 3 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index dc3e93ddd3..ddcec358ac 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -708,9 +708,7 @@ const useAssignTexture = (options: GLTFParserOptions, mapDef?: GLTF.ITextureInfo if (!mapDef) return if (mapDef.texCoord !== undefined && mapDef.texCoord > 0) { - const textureClone = texture.clone() - textureClone.channel = mapDef.texCoord - result.set(textureClone) + texture.channel = mapDef.texCoord } /** @todo properly support extensions */ diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx index 322e96263f..c1f7704d28 100644 --- a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -33,10 +33,12 @@ import { useEntityContext } from '@etherealengine/ecs' import { NO_PROXY, useImmediateEffect } from '@etherealengine/hyperflux' +import createReadableTexture from '@etherealengine/spatial/src/renderer/functions/createReadableTexture' import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTF } from '@gltf-transform/core' import { useEffect, useLayoutEffect } from 'react' import { + CanvasTexture, Color, LinearSRGBColorSpace, MeshPhysicalMaterial, @@ -938,3 +940,176 @@ export const KHRTextureTransformExtensionComponent = defineComponent({ return texture } }) + +export const MozillaHubsLightMapComponent = defineComponent({ + name: 'MozillaHubsLightMapComponent', + jsonID: 'MOZ_lightmap', + + onInit(entity) { + return {} as { + index: 1 + intensity: 1.0 + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.index === 'number') component.index.set(json.index) + if (typeof json.intensity === 'number') component.intensity.set(json.intensity) + }, + + toJSON(entity, component) { + return { + index: component.index.value, + intensity: component.intensity.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, MozillaHubsLightMapComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshPhysicalMaterial + const materialDefinitionComponent = getComponent(entity, MaterialDefinitionComponent) + // Multiply by pi for MeshBasicMaterial shading + const lightMapIntensity = + component.intensity.value * (materialDefinitionComponent.type === 'basic' ? Math.PI : 1.0) + + material.setValues({ lightMapIntensity }) + material.needsUpdate = true + }, [component.intensity.value]) + + const options = getParserOptions(entity) + const lightMap = GLTFLoaderFunctions.useAssignTexture(options, getComponent(entity, MozillaHubsLightMapComponent)) + + useEffect(() => { + if (!lightMap) return + + const material = materialStateComponent.material.value as MeshPhysicalMaterial + + lightMap.channel = 1 + material.lightMap = lightMap + + material.setValues({ lightMap: lightMap }) + material.needsUpdate = true + }, [lightMap]) + + return null + } +}) + +/** + * @deprecated - use KHR_materials_ior and KHR_materials_specular instead + */ +export const KHRMaterialsPBRSpecularGlossinessComponent = defineComponent({ + name: 'KHRMaterialsPBRSpecularGlossinessComponent', + jsonID: 'KHR_materials_pbrSpecularGlossiness', + + onInit(entity) { + return {} as { + diffuseFactor?: [number, number, number, number] + diffuseTexture?: GLTF.ITextureInfo + specularFactor?: [number, number, number] + glossinessFactor?: number + specularGlossinessTexture?: GLTF.ITextureInfo + } + }, + + onSet(entity, component, json) { + if (!json) return + if (Array.isArray(json.diffuseFactor)) component.diffuseFactor.set(json.diffuseFactor) + if (typeof json.diffuseTexture === 'object') component.diffuseTexture.set(json.diffuseTexture) + if (Array.isArray(json.specularFactor)) component.specularFactor.set(json.specularFactor) + if (typeof json.glossinessFactor === 'number') component.glossinessFactor.set(json.glossinessFactor) + if (typeof json.specularGlossinessTexture === 'object') + component.specularGlossinessTexture.set(json.specularGlossinessTexture) + }, + + toJSON(entity, component) { + return { + diffuseFactor: component.diffuseFactor.value, + diffuseTexture: component.diffuseTexture.value, + specularFactor: component.specularFactor.value, + glossinessFactor: component.glossinessFactor.value, + specularGlossinessTexture: component.specularGlossinessTexture.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, KHRMaterialsPBRSpecularGlossinessComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: 'standard' }) + console.warn( + 'KHR_materials_pbrSpecularGlossiness is deprecated. Use KHR_materials_ior and KHR_materials_specular instead.' + ) + }, []) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ + color: new Color().fromArray(component.diffuseFactor.value ?? [1, 1, 1, 1]), + opacity: component.diffuseFactor.value ? component.diffuseFactor.value[3] : 1 + }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.diffuseFactor.value]) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ + roughness: 1 - (component.glossinessFactor.value ?? 1) + }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, component.glossinessFactor.value]) + + const options = getParserOptions(entity) + const map = GLTFLoaderFunctions.useAssignTexture(options, component.diffuseTexture.value) + + useEffect(() => { + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ map }) + material.needsUpdate = true + }, [materialStateComponent.material.value.type, map]) + + const specularGlossinessMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.specularGlossinessTexture.value + ) + + useEffect(() => { + if (!specularGlossinessMap) return + + const abortController = new AbortController() + + invertGlossinessMap(specularGlossinessMap).then((invertedMap) => { + if (abortController.signal.aborted) return + + const material = materialStateComponent.material.value as MeshStandardMaterial + material.setValues({ roughnessMap: invertedMap }) + material.needsUpdate = true + }) + + return () => { + abortController.abort() + } + }, [materialStateComponent.material.value.type, specularGlossinessMap]) + + return null + } +}) + +const invertGlossinessMap = async (glossinessMap: Texture) => { + const mapData: Texture = (await createReadableTexture(glossinessMap, { canvas: true })) as Texture + const canvas = mapData.image as HTMLCanvasElement + const ctx = canvas.getContext('2d')! + ctx.globalCompositeOperation = 'difference' + ctx.fillStyle = 'white' + ctx.fillRect(0, 0, canvas.width, canvas.height) + ctx.globalCompositeOperation = 'source-over' + const invertedTexture = new CanvasTexture(canvas) + return invertedTexture +} From 59cecb9c95c44b9150ed19d2bf66d8d248c2ad99 Mon Sep 17 00:00:00 2001 From: HexaField Date: Tue, 13 Aug 2024 11:24:55 +1000 Subject: [PATCH 32/47] initial support for EE_material --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 21 +-- .../src/gltf/MaterialDefinitionComponent.tsx | 135 ++++++++++++++++-- 2 files changed, 133 insertions(+), 23 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index ddcec358ac..ee7db5ad54 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -27,11 +27,14 @@ import { ComponentType, EntityUUID, UUIDComponent, + getComponent, getOptionalComponent, useOptionalComponent } from '@etherealengine/ecs' import { NO_PROXY, getState, startReactor, useHookstate } from '@etherealengine/hyperflux' +import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' +import { MaterialPrototypeComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTF } from '@gltf-transform/core' import { useEffect } from 'react' import { @@ -57,7 +60,6 @@ import { LoaderUtils, Mesh, MeshBasicMaterial, - MeshPhysicalMaterial, MeshStandardMaterial, NumberKeyframeTrack, QuaternionKeyframeTrack, @@ -450,12 +452,6 @@ export function computeBounds(json: GLTF.IGLTF, geometry: BufferGeometry, primit geometry.boundingSphere = sphere } -const Prototypes = { - basic: MeshBasicMaterial, - standard: MeshStandardMaterial, - physical: MeshPhysicalMaterial -} - /** * Specification: https://github.com/KhronosGroup/glTF/blob/master/specification/2.0/README.md#materials * @param {number} materialIndex @@ -468,8 +464,15 @@ const useLoadMaterial = ( const result = useHookstate(null as null | MeshStandardMaterial | MeshBasicMaterial) useEffect(() => { - const materialTypeValue = Prototypes[materialDef.type] - const material = new materialTypeValue() + /** @todo refactor this into a proper registry, rather than prototype definition entities */ + const materialPrototypeEntity = NameComponent.entitiesByName[materialDef.type]?.[0] + const materialPrototype = materialPrototypeEntity + ? (getComponent(materialPrototypeEntity, MaterialPrototypeComponent).prototypeConstructor as any)[ + materialDef.type + ] + : null + const materialConstructor = materialPrototype ?? MeshStandardMaterial + const material = new materialConstructor() result.set(material) diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx index c1f7704d28..b20f24da0a 100644 --- a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -25,6 +25,7 @@ Ethereal Engine. All Rights Reserved. import { ComponentType, + EntityUUID, UUIDComponent, defineComponent, getComponent, @@ -32,7 +33,7 @@ import { useComponent, useEntityContext } from '@etherealengine/ecs' -import { NO_PROXY, useImmediateEffect } from '@etherealengine/hyperflux' +import { NO_PROXY, startReactor, useImmediateEffect } from '@etherealengine/hyperflux' import createReadableTexture from '@etherealengine/spatial/src/renderer/functions/createReadableTexture' import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTF } from '@gltf-transform/core' @@ -41,6 +42,7 @@ import { CanvasTexture, Color, LinearSRGBColorSpace, + Material, MeshPhysicalMaterial, MeshStandardMaterial, SRGBColorSpace, @@ -55,9 +57,9 @@ export const MaterialDefinitionComponent = defineComponent({ name: 'MaterialDefinitionComponent', onInit: (entity) => { return { - type: 'standard' + type: 'MeshStandardMaterial' } as GLTF.IMaterial & { - type: 'standard' | 'basic' | 'physical' + type: 'MeshStandardMaterial' | 'MeshPhysicalMaterial' | 'MeshBasicMaterial' | (string & {}) } }, @@ -120,7 +122,7 @@ export const KHRUnlitExtensionComponent = defineComponent({ const entity = useEntityContext() useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'basic' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshBasicMaterial' }) }, []) return null @@ -205,7 +207,7 @@ export const KHRClearcoatExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) }, []) useEffect(() => { @@ -310,7 +312,7 @@ export const KHRIridescenceExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) }, []) useEffect(() => { @@ -407,7 +409,7 @@ export const KHRSheenExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ sheen: 1 }) }, []) @@ -491,7 +493,7 @@ export const KHRTransmissionExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) }, []) useEffect(() => { @@ -555,7 +557,7 @@ export const KHRVolumeExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) }, []) useEffect(() => { @@ -629,7 +631,7 @@ export const KHRIorExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) }, []) useEffect(() => { @@ -683,7 +685,7 @@ export const KHRSpecularExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useImmediateEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) }, []) useEffect(() => { @@ -762,7 +764,7 @@ export const EXTBumpExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) }, []) useEffect(() => { @@ -822,7 +824,7 @@ export const KHRAnisotropyExtensionComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'physical' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshPhysicalMaterial' }) }, []) useEffect(() => { @@ -1043,7 +1045,7 @@ export const KHRMaterialsPBRSpecularGlossinessComponent = defineComponent({ const materialStateComponent = useComponent(entity, MaterialStateComponent) useEffect(() => { - setComponent(entity, MaterialDefinitionComponent, { type: 'standard' }) + setComponent(entity, MaterialDefinitionComponent, { type: 'MeshStandardMaterial' }) console.warn( 'KHR_materials_pbrSpecularGlossiness is deprecated. Use KHR_materials_ior and KHR_materials_specular instead.' ) @@ -1113,3 +1115,108 @@ const invertGlossinessMap = async (glossinessMap: Texture) => { const invertedTexture = new CanvasTexture(canvas) return invertedTexture } + +export type MaterialExtensionPluginType = { id: string; uniforms: { [key: string]: any } } + +export const EEMaterialComponent = defineComponent({ + name: 'EEMaterialComponent', + jsonID: 'EE_material', + + onInit(entity) { + return {} as { + uuid: EntityUUID + name: string + prototype: string + args: { + [field: string]: { + type: string + contents: any + } + } + plugins: MaterialExtensionPluginType[] + } + }, + + onSet(entity, component, json) { + if (!json) return + if (typeof json.uuid === 'string') component.uuid.set(json.uuid) + if (typeof json.name === 'string') component.name.set(json.name) + if (typeof json.prototype === 'string') component.prototype.set(json.prototype) + if (typeof json.args === 'object') component.args.set(json.args) + if (Array.isArray(json.plugins)) component.plugins.set(json.plugins) + }, + + toJSON(entity, component) { + return { + uuid: component.uuid.value, + name: component.name.value, + prototype: component.prototype.value, + args: component.args.value, + plugins: component.plugins.value + } + }, + + reactor: () => { + const entity = useEntityContext() + const component = useComponent(entity, EEMaterialComponent) + const materialStateComponent = useComponent(entity, MaterialStateComponent) + + useEffect(() => { + setComponent(entity, MaterialDefinitionComponent, { type: component.prototype.value }) + }, [component.prototype.value]) + + useEffect(() => { + const options = getParserOptions(entity) + const resultProperties = {} as Record + const texturePromises = Object.fromEntries( + Object.entries(component.args.value).filter(([k, v]) => v.type === 'texture' && v.contents) + ) + + const reactor = startReactor(() => { + for (const [k, v] of Object.entries(component.args.value)) { + if (v.type === 'texture') { + if (v.contents) { + const texture = GLTFLoaderFunctions.useAssignTexture(options, v.contents) + useEffect(() => { + if (!texture) return + if (k === 'map') texture.colorSpace = SRGBColorSpace + resultProperties[k] = texture + delete texturePromises[k] + if (Object.keys(texturePromises).length === 0) { + const material = materialStateComponent.material.value as Material + material.setValues(resultProperties) + material.needsUpdate = true + reactor.stop() + } + }, [texture]) + } else { + useEffect(() => { + resultProperties[k] = null + }, []) + } + } else if (v.type === 'color') { + useEffect(() => { + if (v.contents !== null && !(v.contents as Color)?.isColor) { + resultProperties[k] = new Color(v.contents) + } else { + resultProperties[k] = v.contents + } + }, []) + } else { + useEffect(() => { + resultProperties[k] = v.contents + }, []) + } + } + + return null + }) + + return () => { + reactor.stop() + } + }, [materialStateComponent.material.type.value, component.args.value]) + + return null + } +}) From b246074e9754f63ee4f05b1825f2b4684585413d Mon Sep 17 00:00:00 2001 From: HexaField Date: Tue, 13 Aug 2024 11:27:23 +1000 Subject: [PATCH 33/47] console logs --- packages/engine/src/gltf/GLTFExtensions.ts | 1 - packages/engine/src/gltf/GLTFState.tsx | 2 -- 2 files changed, 3 deletions(-) diff --git a/packages/engine/src/gltf/GLTFExtensions.ts b/packages/engine/src/gltf/GLTFExtensions.ts index 2bce8a8767..f6142b6da7 100644 --- a/packages/engine/src/gltf/GLTFExtensions.ts +++ b/packages/engine/src/gltf/GLTFExtensions.ts @@ -116,7 +116,6 @@ export const EXT_MESHOPT_COMPRESSION = { extensionDef.buffer, (bufferView: ArrayBuffer) => new Promise((resolve, reject) => { - console.log('bufferView', bufferView) const json = options.document const byteOffset = extensionDef.byteOffset || 0 diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 5ae8ccb34c..51ff87d0ee 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -1104,8 +1104,6 @@ export const MorphTargetReactor = (props: { mesh.morphTargetInfluences[i].set(meshDef.weights[i]) } } - - console.log('Morph targets loaded', mesh) }, [morphTargets]) return null From dccc85f333c9cd3803d596110f8f6149ac46acca Mon Sep 17 00:00:00 2001 From: HexaField Date: Tue, 13 Aug 2024 11:38:38 +1000 Subject: [PATCH 34/47] assign texture bug fixes --- .../src/gltf/MaterialDefinitionComponent.tsx | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx index b20f24da0a..48c341519f 100644 --- a/packages/engine/src/gltf/MaterialDefinitionComponent.tsx +++ b/packages/engine/src/gltf/MaterialDefinitionComponent.tsx @@ -225,7 +225,7 @@ export const KHRClearcoatExtensionComponent = defineComponent({ }, [materialStateComponent.material.value.type, component.clearcoatRoughnessFactor.value]) const options = getParserOptions(entity) - const clearcoatMap = GLTFLoaderFunctions.useAssignTexture(options, component.clearcoatTexture.value) + const clearcoatMap = GLTFLoaderFunctions.useAssignTexture(options, component.clearcoatTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial @@ -235,7 +235,7 @@ export const KHRClearcoatExtensionComponent = defineComponent({ const clearcoatRoughnessMap = GLTFLoaderFunctions.useAssignTexture( options, - component.clearcoatRoughnessTexture.value + component.clearcoatRoughnessTexture.get(NO_PROXY) ) useEffect(() => { @@ -244,7 +244,10 @@ export const KHRClearcoatExtensionComponent = defineComponent({ material.needsUpdate = true }, [materialStateComponent.material.value.type, clearcoatRoughnessMap]) - const clearcoatNormalMap = GLTFLoaderFunctions.useAssignTexture(options, component.clearcoatNormalTexture.value) + const clearcoatNormalMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.clearcoatNormalTexture.get(NO_PROXY) + ) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial @@ -330,7 +333,7 @@ export const KHRIridescenceExtensionComponent = defineComponent({ }, [materialStateComponent.material.value.type, component.iridescenceIor.value]) const options = getParserOptions(entity) - const iridescenceMap = GLTFLoaderFunctions.useAssignTexture(options, component.iridescenceTexture.value) + const iridescenceMap = GLTFLoaderFunctions.useAssignTexture(options, component.iridescenceTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial @@ -355,7 +358,7 @@ export const KHRIridescenceExtensionComponent = defineComponent({ const iridescenceThicknessMap = GLTFLoaderFunctions.useAssignTexture( options, - component.iridescenceThicknessTexture.value + component.iridescenceThicknessTexture.get(NO_PROXY) ) useEffect(() => { @@ -436,7 +439,7 @@ export const KHRSheenExtensionComponent = defineComponent({ }, [materialStateComponent.material.value.type, component.sheenRoughnessFactor.value]) const options = getParserOptions(entity) - const sheenColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.sheenColorTexture.value) + const sheenColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.sheenColorTexture.get(NO_PROXY)) useEffect(() => { if (sheenColorMap) sheenColorMap.colorSpace = SRGBColorSpace @@ -445,7 +448,10 @@ export const KHRSheenExtensionComponent = defineComponent({ material.needsUpdate = true }, [materialStateComponent.material.value.type, sheenColorMap]) - const sheenRoughnessMap = GLTFLoaderFunctions.useAssignTexture(options, component.sheenRoughnessTexture.value) + const sheenRoughnessMap = GLTFLoaderFunctions.useAssignTexture( + options, + component.sheenRoughnessTexture.get(NO_PROXY) + ) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial @@ -501,16 +507,16 @@ export const KHRTransmissionExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ transmission: component.transmissionFactor.value }) material.needsUpdate = true - }, [materialStateComponent.material.value.type, component.transmissionFactor.value]) + }, [materialStateComponent.material.value, component.transmissionFactor.value]) const options = getParserOptions(entity) - const transmissionMap = GLTFLoaderFunctions.useAssignTexture(options, component.transmissionTexture.value) + const transmissionMap = GLTFLoaderFunctions.useAssignTexture(options, component.transmissionTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ transmissionMap }) material.needsUpdate = true - }, [materialStateComponent.material.value.type, transmissionMap]) + }, [materialStateComponent.material.value, transmissionMap]) return null } @@ -587,7 +593,7 @@ export const KHRVolumeExtensionComponent = defineComponent({ }, [materialStateComponent.material.value.type, component.attenuationColorFactor.value]) const options = getParserOptions(entity) - const thicknessMap = GLTFLoaderFunctions.useAssignTexture(options, component.thicknessTexture.value) + const thicknessMap = GLTFLoaderFunctions.useAssignTexture(options, component.thicknessTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial @@ -692,7 +698,7 @@ export const KHRSpecularExtensionComponent = defineComponent({ const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ specularIntensity: component.specularFactor.value ?? 1.0 }) material.needsUpdate = true - }, [materialStateComponent.material.value.type, component.specularFactor.value]) + }, [materialStateComponent.material.type.value, component.specularFactor.value]) useEffect(() => { const specularColorFactor = component.specularColorFactor.value ?? [1, 1, 1] @@ -706,24 +712,24 @@ export const KHRSpecularExtensionComponent = defineComponent({ ) }) material.needsUpdate = true - }, [materialStateComponent.material.value.type, component.specularColorFactor.value]) + }, [materialStateComponent.material.type.value, component.specularColorFactor.value]) const options = getParserOptions(entity) - const specularMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularTexture.value) + const specularMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ specularIntensityMap: specularMap }) material.needsUpdate = true - }, [materialStateComponent.material.value.type, specularMap]) + }, [materialStateComponent.material.type.value, specularMap]) - const specularColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularColorTexture.value) + const specularColorMap = GLTFLoaderFunctions.useAssignTexture(options, component.specularColorTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial material.setValues({ specularColorMap }) material.needsUpdate = true - }, [materialStateComponent.material.value.type, specularColorMap]) + }, [materialStateComponent.material.type.value, specularColorMap]) return null } @@ -774,7 +780,7 @@ export const EXTBumpExtensionComponent = defineComponent({ }, [materialStateComponent.material.value.type, component.bumpFactor.value]) const options = getParserOptions(entity) - const bumpMap = GLTFLoaderFunctions.useAssignTexture(options, component.bumpTexture.value) + const bumpMap = GLTFLoaderFunctions.useAssignTexture(options, component.bumpTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial @@ -840,7 +846,7 @@ export const KHRAnisotropyExtensionComponent = defineComponent({ }, [materialStateComponent.material.value.type, component.anisotropyRotation.value]) const options = getParserOptions(entity) - const anisotropyMap = GLTFLoaderFunctions.useAssignTexture(options, component.anisotropyTexture.value) + const anisotropyMap = GLTFLoaderFunctions.useAssignTexture(options, component.anisotropyTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshPhysicalMaterial @@ -984,7 +990,7 @@ export const MozillaHubsLightMapComponent = defineComponent({ }, [component.intensity.value]) const options = getParserOptions(entity) - const lightMap = GLTFLoaderFunctions.useAssignTexture(options, getComponent(entity, MozillaHubsLightMapComponent)) + const lightMap = GLTFLoaderFunctions.useAssignTexture(options, component.get(NO_PROXY)) useEffect(() => { if (!lightMap) return @@ -1069,7 +1075,7 @@ export const KHRMaterialsPBRSpecularGlossinessComponent = defineComponent({ }, [materialStateComponent.material.value.type, component.glossinessFactor.value]) const options = getParserOptions(entity) - const map = GLTFLoaderFunctions.useAssignTexture(options, component.diffuseTexture.value) + const map = GLTFLoaderFunctions.useAssignTexture(options, component.diffuseTexture.get(NO_PROXY)) useEffect(() => { const material = materialStateComponent.material.value as MeshStandardMaterial @@ -1079,7 +1085,7 @@ export const KHRMaterialsPBRSpecularGlossinessComponent = defineComponent({ const specularGlossinessMap = GLTFLoaderFunctions.useAssignTexture( options, - component.specularGlossinessTexture.value + component.specularGlossinessTexture.get(NO_PROXY) ) useEffect(() => { From 9cb99773e418125b9193943edbb947177284d824 Mon Sep 17 00:00:00 2001 From: HexaField Date: Wed, 14 Aug 2024 15:21:01 +1000 Subject: [PATCH 35/47] Support nested GLTFComponent loading, bug fixes with materials and multiple primitives --- packages/client-core/src/world/Location.tsx | 3 +- .../editor/src/functions/sceneFunctions.tsx | 6 +- packages/engine/src/gltf/GLTFComponent.tsx | 59 ++++++---- .../engine/src/gltf/GLTFLoaderFunctions.ts | 12 +- packages/engine/src/gltf/GLTFState.tsx | 104 ++++++++++++------ .../src/scene/components/ModelComponent.tsx | 2 +- .../renderer/materials/MaterialComponent.tsx | 23 ++-- 7 files changed, 134 insertions(+), 75 deletions(-) diff --git a/packages/client-core/src/world/Location.tsx b/packages/client-core/src/world/Location.tsx index 08104a0e86..f21987fe1b 100755 --- a/packages/client-core/src/world/Location.tsx +++ b/packages/client-core/src/world/Location.tsx @@ -71,7 +71,8 @@ const LocationPage = ({ online }: Props) => { useLoadEngineWithScene() useEffect(() => { - if (ready.value) logger.info({ event_name: 'enter_location' }) + if (!ready.value) return + logger.info({ event_name: 'enter_location' }) return () => logger.info({ event_name: 'exit_location' }) }, [ready.value]) diff --git a/packages/editor/src/functions/sceneFunctions.tsx b/packages/editor/src/functions/sceneFunctions.tsx index 6ee490ff1c..e007dc71c2 100644 --- a/packages/editor/src/functions/sceneFunctions.tsx +++ b/packages/editor/src/functions/sceneFunctions.tsx @@ -29,8 +29,8 @@ import config from '@etherealengine/common/src/config' import multiLogger from '@etherealengine/common/src/logger' import { StaticResourceType, fileBrowserPath, staticResourcePath } from '@etherealengine/common/src/schema.type.module' import { cleanString } from '@etherealengine/common/src/utils/cleanString' -import { EntityUUID, UUIDComponent, UndefinedEntity } from '@etherealengine/ecs' -import { getComponent, setComponent } from '@etherealengine/ecs/src/ComponentFunctions' +import { EntityUUID, UndefinedEntity } from '@etherealengine/ecs' +import { setComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { Engine } from '@etherealengine/ecs/src/Engine' import { GLTFComponent } from '@etherealengine/engine/src/gltf/GLTFComponent' import { GLTFDocumentState } from '@etherealengine/engine/src/gltf/GLTFDocumentState' @@ -93,7 +93,7 @@ export const saveSceneGLTF = async ( if (signal.aborted) throw new Error(i18n.t('editor:errors.saveProjectAborted')) const { rootEntity } = getState(EditorState) - const sourceID = `${getComponent(rootEntity, UUIDComponent)}-${getComponent(rootEntity, GLTFComponent).src}` + const sourceID = GLTFComponent.getInstanceID(rootEntity) const sceneName = cleanString(sceneFile!.replace('.scene.json', '').replace('.gltf', '')) const currentSceneDirectory = getState(EditorState).scenePath!.split('/').slice(0, -1).join('/') diff --git a/packages/engine/src/gltf/GLTFComponent.tsx b/packages/engine/src/gltf/GLTFComponent.tsx index bdc738bb7e..06925656fe 100644 --- a/packages/engine/src/gltf/GLTFComponent.tsx +++ b/packages/engine/src/gltf/GLTFComponent.tsx @@ -30,7 +30,6 @@ import { parseStorageProviderURLs } from '@etherealengine/common/src/utils/parse import { defineComponent, Entity, - EntityUUID, getComponent, getMutableComponent, getOptionalComponent, @@ -40,7 +39,14 @@ import { useQuery, UUIDComponent } from '@etherealengine/ecs' -import { dispatchAction, getState, useHookstate, useMutableState } from '@etherealengine/hyperflux' +import { + dispatchAction, + getMutableState, + getState, + none, + useHookstate, + useMutableState +} from '@etherealengine/hyperflux' import { FileLoader } from '../assets/loaders/base/FileLoader' import { @@ -48,15 +54,16 @@ import { BINARY_EXTENSION_HEADER_LENGTH, BINARY_EXTENSION_HEADER_MAGIC } from '../assets/loaders/gltf/GLTFExtensions' -import { ModelComponent } from '../scene/components/ModelComponent' import { SourceComponent } from '../scene/components/SourceComponent' import { SceneJsonType } from '../scene/types/SceneTypes' import { migrateSceneJSONToGLTF } from './convertJsonToGLTF' import { GLTFDocumentState, GLTFSnapshotAction } from './GLTFDocumentState' +import { GLTFSourceState } from './GLTFState' import { ResourcePendingComponent } from './ResourcePendingComponent' export const GLTFComponent = defineComponent({ name: 'GLTFComponent', + jsonID: 'EE_model', onInit(entity) { return { @@ -77,9 +84,20 @@ export const GLTFComponent = defineComponent({ useGLTFDocument(gltfComponent.src.value, entity) - const documentID = useComponent(entity, SourceComponent).value + const sourceID = GLTFComponent.getInstanceID(entity) + + useEffect(() => { + getMutableState(GLTFSourceState)[sourceID].set(entity) + return () => { + getMutableState(GLTFSourceState)[sourceID].set(none) + } + }, []) - return + return + }, + + getInstanceID: (entity) => { + return `${getComponent(entity, UUIDComponent)}-${getComponent(entity, GLTFComponent).src}` } }) @@ -91,19 +109,19 @@ const ResourceReactor = (props: { documentID: string; entity: Entity }) => { useEffect(() => { if (getComponent(props.entity, GLTFComponent).progress === 100) return if (!getState(GLTFDocumentState)[props.documentID]) return - const document = getState(GLTFDocumentState)[props.documentID] - const modelNodes = document.nodes?.filter((node) => !!node.extensions?.[ModelComponent.jsonID]) - if (modelNodes) { - for (const node of modelNodes) { - //check if an entity exists for this node, and has a model component - const uuid = node.extensions![UUIDComponent.jsonID] as EntityUUID - if (!UUIDComponent.entitiesByUUIDState[uuid]) return - const entity = UUIDComponent.entitiesByUUIDState[uuid].value - const model = getOptionalComponent(entity, ModelComponent) - //ensure that model contents have been loaded into the scene - if (!model?.scene) return - } - } + // const document = getState(GLTFDocumentState)[props.documentID] + // const modelNodes = document.nodes?.filter((node) => !!node.extensions?.[ModelComponent.jsonID]) + // if (modelNodes) { + // for (const node of modelNodes) { + // //check if an entity exists for this node, and has a model component + // const uuid = node.extensions![UUIDComponent.jsonID] as EntityUUID + // if (!UUIDComponent.entitiesByUUIDState[uuid]) return + // const entity = UUIDComponent.entitiesByUUIDState[uuid].value + // const model = getOptionalComponent(entity, ModelComponent) + // //ensure that model contents have been loaded into the scene + // if (!model?.scene) return + // } + // } const entities = resourceQuery.filter((e) => getComponent(e, SourceComponent) === props.documentID) if (!entities.length) { getMutableComponent(props.entity, GLTFComponent).progress.set(100) @@ -144,10 +162,9 @@ const onProgress: (event: ProgressEvent) => void = (event) => { const useGLTFDocument = (url: string, entity: Entity) => { const state = useComponent(entity, GLTFComponent) - const sourceComponent = useComponent(entity, SourceComponent) + const source = GLTFComponent.getInstanceID(entity) useEffect(() => { - const source = sourceComponent.value return () => { dispatchAction(GLTFSnapshotAction.unload({ source })) } @@ -193,7 +210,7 @@ const useGLTFDocument = (url: string, entity: Entity) => { dispatchAction( GLTFSnapshotAction.createSnapshot({ - source: getComponent(entity, SourceComponent), + source, data: parseStorageProviderURLs(JSON.parse(JSON.stringify(json))) }) ) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index ee7db5ad54..31c374e4dc 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -529,7 +529,7 @@ const useLoadMaterial = ( const metalnessMap = GLTFLoaderFunctions.useAssignTexture( options, - materialDef.type === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture ) useEffect(() => { @@ -540,7 +540,7 @@ const useLoadMaterial = ( const roughnessMap = GLTFLoaderFunctions.useAssignTexture( options, - materialDef.type === 'basic' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.pbrMetallicRoughness?.metallicRoughnessTexture ) useEffect(() => { @@ -576,7 +576,7 @@ const useLoadMaterial = ( const normalMap = GLTFLoaderFunctions.useAssignTexture( options, - materialDef.type === 'basic' ? undefined : materialDef.normalTexture + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.normalTexture ) useEffect(() => { @@ -597,7 +597,7 @@ const useLoadMaterial = ( const aoMap = GLTFLoaderFunctions.useAssignTexture( options, - materialDef.type === 'basic' ? undefined : materialDef.occlusionTexture + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.occlusionTexture ) useEffect(() => { @@ -623,7 +623,7 @@ const useLoadMaterial = ( const emissiveMap = GLTFLoaderFunctions.useAssignTexture( options, - materialDef.type === 'basic' ? undefined : materialDef.emissiveTexture + materialDef.type === 'MeshBasicMaterial' ? undefined : materialDef.emissiveTexture ) useEffect(() => { @@ -769,7 +769,7 @@ const useLoadTexture = (options: GLTFParserOptions, textureIndex?: number) => { /** @todo properly support texture extensions, this is a hack */ const sourceIndex = - (extensions && Object.values(extensions).find((ext) => typeof ext.source === 'number')?.source) || + (extensions && Object.values(extensions).find((ext) => typeof ext.source === 'number')?.source) ?? textureDef?.source! const sourceDef = typeof sourceIndex === 'number' ? json.images![sourceIndex] : null diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 51ff87d0ee..6aa73bc750 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -150,13 +150,10 @@ export const GLTFSourceState = defineState({ setComponent(entity, Object3DComponent, obj3d) addObjectToGroup(entity, obj3d) proxifyParentChildRelationships(obj3d) - getMutableState(GLTFSourceState)[sourceID].set(entity) return entity }, unload: (entity: Entity) => { - const sourceID = `${getComponent(entity, UUIDComponent)}-${getComponent(entity, GLTFComponent).src}` - getMutableState(GLTFSourceState)[sourceID].set(none) removeEntity(entity) } }) @@ -387,11 +384,18 @@ const ChildGLTFReactor = (props: { source: string }) => { const source = props.source const index = useHookstate(getMutableState(GLTFSnapshotState)[source].index).value - const entity = useHookstate(getMutableState(GLTFSourceState)[source]).value const parentUUID = useComponent(entity, UUIDComponent).value useLayoutEffect(() => { + return () => { + getMutableState(GLTFDocumentState)[source].set(none) + getMutableState(GLTFNodeState)[source].set(none) + } + }, []) + + useLayoutEffect(() => { + const index = getState(GLTFSnapshotState)[source].index // update the modified state if (index > 0) getMutableState(GLTFModifiedState)[source].set(true) @@ -404,25 +408,23 @@ const ChildGLTFReactor = (props: { source: string }) => { getMutableState(GLTFNodeState)[source].set(nodesDictionary) }, [index]) - useLayoutEffect(() => { - return () => { - getMutableState(GLTFDocumentState)[source].set(none) - getMutableState(GLTFNodeState)[source].set(none) - } - }, []) + const nodeState = useHookstate(getMutableState(GLTFNodeState))[source] + const documentState = useMutableState(GLTFDocumentState)[source] + const physicsWorld = Physics.useWorld(entity) + + if (!physicsWorld || !documentState.value || !nodeState.value) return null return } export const DocumentReactor = (props: { documentID: string; parentUUID: EntityUUID }) => { - const nodeState = useHookstate(getMutableState(GLTFNodeState)[props.documentID]) - const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const nodeState = useHookstate(getMutableState(GLTFNodeState))[props.documentID] + const documentState = useMutableState(GLTFDocumentState)[props.documentID] const animationState = useHookstate([] as AnimationClip[]) const rootEntity = UUIDComponent.useEntityByUUID(props.parentUUID) useEffect(() => { - if (!documentState.value || !nodeState.value || animationState.length !== documentState.value.animations?.length) - return + if (animationState.length !== documentState.value.animations?.length) return const scene = getComponent(rootEntity, Object3DComponent) scene.animations = animationState.get(NO_PROXY) as AnimationClip[] @@ -436,8 +438,6 @@ export const DocumentReactor = (props: { documentID: string; parentUUID: EntityU } }, [animationState]) - if (!documentState.value || !nodeState.value) return null - return ( <> {Object.entries(nodeState.get(NO_PROXY)).map(([uuid, { nodeIndex, childIndex, parentUUID }]) => ( @@ -595,8 +595,7 @@ const ParentNodeReactor = (props: { documentID: string }) => { const parentEntity = UUIDComponent.useEntityByUUID(props.parentUUID) - const physicsWorld = Physics.useWorld(parentEntity) - if (!parentEntity || !physicsWorld) return null + if (!parentEntity) return null return } @@ -805,7 +804,7 @@ const ExtensionReactor = (props: { entity: Entity; extension: string; nodeIndex: } const MeshReactor = (props: { nodeIndex: number; documentID: string; entity: Entity }) => { - const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) + const documentState = useMutableState(GLTFDocumentState)[props.documentID] const nodes = documentState.nodes!.get(NO_PROXY)! const node = nodes[props.nodeIndex]! @@ -815,11 +814,14 @@ const MeshReactor = (props: { nodeIndex: number; documentID: string; entity: Ent setComponent(props.entity, VisibleComponent) }, []) + const isSinglePrimitive = mesh.primitives.length === 1 + return ( <> {mesh.primitives.map((primitive, index) => ( { +const PrimitiveReactor = (props: { + isSinglePrimitive: boolean + primitiveIndex: number + nodeIndex: number + documentID: string + entity: Entity +}) => { const documentState = useHookstate(getMutableState(GLTFDocumentState)[props.documentID]) const nodes = documentState.nodes!.get(NO_PROXY)! - const node = nodes[props.nodeIndex]! + const node = nodes[props.nodeIndex]! as GLTF.INode const meshDef = documentState.meshes.get(NO_PROXY)![node.mesh!] @@ -951,6 +959,33 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do const options = getParserOptions(props.entity) const geometry = GLTFLoaderFunctions.useLoadPrimitive(options, props.nodeIndex, props.primitiveIndex) + /** @todo until we have multiple geometry to a single mesh entity support, we have to make children */ + const entity = useHookstate(() => { + if (props.isSinglePrimitive) return props.entity + + const uuid = (getNodeUUID(node, props.documentID, props.nodeIndex) + + '-primitive-' + + props.primitiveIndex) as EntityUUID + const entity = UUIDComponent.getOrCreateEntityByUUID(uuid) + + setComponent(entity, UUIDComponent, uuid) + setComponent(entity, SourceComponent, props.documentID) + + /** Ensure all base components are added for synchronous mount */ + setComponent(entity, EntityTreeComponent, { parentEntity: props.entity, childIndex: props.primitiveIndex }) + setComponent(entity, NameComponent, (node.name ?? 'Node-' + props.nodeIndex) + '_' + props.primitiveIndex) + setComponent(entity, TransformComponent) + setComponent(entity, VisibleComponent) + + return entity + }).value + + useEffect(() => { + return () => { + if (!props.isSinglePrimitive) removeEntity(entity) + } + }, []) + useLayoutEffect(() => { if (!geometry) return @@ -959,20 +994,20 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do const skinnedMesh = new SkinnedMesh(geometry, new MeshBasicMaterial()) mesh = skinnedMesh skinnedMesh.skeleton = new Skeleton() - setComponent(props.entity, MeshComponent, skinnedMesh) - setComponent(props.entity, SkinnedMeshComponent, skinnedMesh) + setComponent(entity, MeshComponent, skinnedMesh) + setComponent(entity, SkinnedMeshComponent, skinnedMesh) } else { mesh = new Mesh(geometry, new MeshBasicMaterial()) - setComponent(props.entity, MeshComponent, mesh) + setComponent(entity, MeshComponent, mesh) } /** @todo multiple primitive support */ - addObjectToGroup(props.entity, mesh) + addObjectToGroup(entity, mesh) return () => { - removeComponent(props.entity, SkinnedMeshComponent) - removeComponent(props.entity, MeshComponent) - removeObjectFromGroup(props.entity, mesh) + removeComponent(entity, SkinnedMeshComponent) + removeComponent(entity, MeshComponent) + removeObjectFromGroup(entity, mesh) } }, [node.skin, geometry]) @@ -985,7 +1020,7 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do nodeIndex={props.nodeIndex} primitiveIndex={props.primitiveIndex} documentID={props.documentID} - entity={props.entity} + entity={entity} /> )} {typeof primitive.material === 'number' && ( @@ -993,14 +1028,14 @@ const PrimitiveReactor = (props: { primitiveIndex: number; nodeIndex: number; do nodeIndex={props.nodeIndex} primitiveIndex={props.primitiveIndex} documentID={props.documentID} - entity={props.entity} + entity={entity} /> )} {primitive.targets && ( { const gltfEntity = getAncestorWithComponent(entity, GLTFComponent) - const document = getState(GLTFDocumentState)[getComponent(gltfEntity, SourceComponent)] + const documentID = GLTFComponent.getInstanceID(gltfEntity) const gltfComponent = getComponent(gltfEntity, GLTFComponent) + const document = getState(GLTFDocumentState)[documentID] const gltfLoader = getState(AssetLoaderState).gltfLoader return { document, - documentID: getComponent(gltfEntity, SourceComponent), + documentID, url: gltfComponent.src, path: LoaderUtils.extractUrlBase(gltfComponent.src), body: gltfComponent.body, diff --git a/packages/engine/src/scene/components/ModelComponent.tsx b/packages/engine/src/scene/components/ModelComponent.tsx index fd8297b382..b50ecf6565 100644 --- a/packages/engine/src/scene/components/ModelComponent.tsx +++ b/packages/engine/src/scene/components/ModelComponent.tsx @@ -70,7 +70,7 @@ import { SourceComponent } from './SourceComponent' */ export const ModelComponent = defineComponent({ name: 'ModelComponent', - jsonID: 'EE_model', + // jsonID: 'EE_model', onInit: (entity) => { return { diff --git a/packages/spatial/src/renderer/materials/MaterialComponent.tsx b/packages/spatial/src/renderer/materials/MaterialComponent.tsx index 0366bf7699..365d14eb2a 100644 --- a/packages/spatial/src/renderer/materials/MaterialComponent.tsx +++ b/packages/spatial/src/renderer/materials/MaterialComponent.tsx @@ -23,7 +23,7 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { Material, Mesh, Shader, WebGLRenderer } from 'three' +import { Material, Shader, WebGLRenderer } from 'three' import { Component, @@ -155,19 +155,21 @@ export const MaterialInstanceComponent = defineComponent({ reactor: () => { const entity = useEntityContext() const materialComponent = useComponent(entity, MaterialInstanceComponent) - const meshComponent = useComponent(entity, MeshComponent) - if (Array.isArray(meshComponent.material.value)) + if (materialComponent.uuid.value.length === 0) return null + + if (materialComponent.uuid.value.length > 1) return ( <> {materialComponent.uuid.value.map((uuid, index) => ( - + ))} ) return ( { +const MaterialInstanceSubReactor = (props: { array: boolean; uuid: EntityUUID; entity: Entity; index: number }) => { const { uuid, entity, index } = props const materialStateEntity = UUIDComponent.useEntityByUUID(uuid) const materialStateComponent = useComponent(materialStateEntity, MaterialStateComponent) const meshComponent = useComponent(entity, MeshComponent) useEffect(() => { - const mesh = meshComponent.value as Mesh - const material = materialStateComponent.material.value as Material - if (Array.isArray(mesh.material)) mesh.material[index] = material - else mesh.material = material + const material = getComponent(materialStateEntity, MaterialStateComponent).material + if (props.array) { + if (!Array.isArray(meshComponent.material.value)) meshComponent.material.set([]) + meshComponent.material[index].set(material) + } else { + meshComponent.material.set(material) + } }, [materialStateComponent.material]) return null From f867d8f3b11f18a1978cc0ee34b1e45be28d1bf6 Mon Sep 17 00:00:00 2001 From: HexaField Date: Wed, 14 Aug 2024 22:03:38 +1000 Subject: [PATCH 36/47] working on loading avatars --- .../components/AvatarAnimationComponent.ts | 213 ++++++++++++++++-- .../src/avatar/state/AvatarNetworkState.tsx | 6 +- packages/engine/src/gltf/GLTFState.tsx | 3 +- 3 files changed, 198 insertions(+), 24 deletions(-) diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index 39d6d3bde5..12e6783c2f 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -23,21 +23,32 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { VRM, VRMHumanBoneName, VRMHumanBones } from '@pixiv/three-vrm' +import { + VRM, + VRM1Meta, + VRMHumanBone, + VRMHumanBoneName, + VRMHumanBones, + VRMHumanoid, + VRMParameters +} from '@pixiv/three-vrm' +import type * as V0VRM from '@pixiv/types-vrm-0.0' + import { useEffect } from 'react' -import { AnimationAction, Group, Matrix4, SkeletonHelper, Vector3 } from 'three' +import { AnimationAction, Euler, Group, Matrix4, SkeletonHelper, Vector3 } from 'three' import { defineComponent, getComponent, + hasComponent, removeComponent, setComponent, useComponent, useOptionalComponent } from '@etherealengine/ecs/src/ComponentFunctions' -import { Entity } from '@etherealengine/ecs/src/Entity' -import { createEntity, entityExists, removeEntity, useEntityContext } from '@etherealengine/ecs/src/EntityFunctions' -import { getMutableState, matches, none, useHookstate } from '@etherealengine/hyperflux' +import { Entity, EntityUUID } from '@etherealengine/ecs/src/Entity' +import { createEntity, removeEntity, useEntityContext } from '@etherealengine/ecs/src/EntityFunctions' +import { getMutableState, getState, matches, none, useHookstate } from '@etherealengine/hyperflux' import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' import { setObjectLayers } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' @@ -46,9 +57,16 @@ import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/Obj import { RendererState } from '@etherealengine/spatial/src/renderer/RendererState' import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' -import { ModelComponent } from '../../scene/components/ModelComponent' +import { UUIDComponent } from '@etherealengine/ecs' +import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' +import { EntityTreeComponent, iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { GLTF } from '@gltf-transform/core' +import { GLTFComponent } from '../../gltf/GLTFComponent' +import { GLTFDocumentState } from '../../gltf/GLTFDocumentState' +import { proxifyParentChildRelationships } from '../../scene/functions/loadGLTFModel' import { preloadedAnimations } from '../animation/Util' import { AnimationState } from '../AnimationManager' +import { mixamoVRMRigMap } from '../AvatarBoneMatching' import { retargetAvatarAnimations, setAvatarSpeedFromRootMotion, @@ -58,6 +76,7 @@ import { import { AvatarState } from '../state/AvatarNetworkState' import { AvatarComponent } from './AvatarComponent' import { AvatarPendingComponent } from './AvatarPendingComponent' +import { BoneComponent } from './BoneComponent' export const AvatarAnimationComponent = defineComponent({ name: 'AvatarAnimationComponent', @@ -91,6 +110,7 @@ export const AvatarAnimationComponent = defineComponent({ }) export type Matrices = { local: Matrix4; world: Matrix4 } + export const AvatarRigComponent = defineComponent({ name: 'AvatarRigComponent', @@ -128,7 +148,7 @@ export const AvatarRigComponent = defineComponent({ const rigComponent = useComponent(entity, AvatarRigComponent) const pending = useOptionalComponent(entity, AvatarPendingComponent) const visible = useOptionalComponent(entity, VisibleComponent) - const modelComponent = useOptionalComponent(entity, ModelComponent) + const gltfComponent = useOptionalComponent(entity, GLTFComponent) const locomotionAnimationState = useHookstate( getMutableState(AnimationState).loadedAnimations[preloadedAnimations.locomotion] ) @@ -162,22 +182,26 @@ export const AvatarRigComponent = defineComponent({ } }, [visible, debugEnabled, pending, rigComponent.normalizedRig]) + /** @todo move asset loading to a new VRMComponent */ useEffect(() => { - if (!modelComponent?.asset?.value) return - const model = getComponent(entity, ModelComponent) - setupAvatarProportions(entity, model.asset as VRM) - setComponent(entity, AvatarRigComponent, { - vrm: model.asset as VRM, - avatarURL: model.src - }) + if (!rigComponent?.avatarURL?.value) return + setComponent(entity, GLTFComponent, { src: rigComponent.avatarURL.value }) return () => { - if (!entityExists(entity)) return - setComponent(entity, AvatarRigComponent, { - vrm: null!, - avatarURL: null - }) + removeComponent(entity, GLTFComponent) } - }, [modelComponent?.asset]) + }, [rigComponent?.avatarURL?.value]) + + useEffect(() => { + if (gltfComponent?.progress?.value !== 100) return + const vrm = createVRM(entity) + rigComponent.vrm.set(vrm) + }, [gltfComponent?.progress?.value]) + + useEffect(() => { + if (!rigComponent?.vrm?.value) return + const rig = getComponent(entity, AvatarRigComponent) + setupAvatarProportions(entity, rig.vrm as VRM) + }, [rigComponent?.vrm?.value]) useEffect(() => { if ( @@ -205,3 +229,152 @@ export const AvatarRigComponent = defineComponent({ return null } }) + +const _rightHandPos = new Vector3(), + _rightUpperArmPos = new Vector3() + +export default function createVRM(rootEntity: Entity) { + const documentID = GLTFComponent.getInstanceID(rootEntity) + const gltf = getState(GLTFDocumentState)[documentID] + + if (!hasComponent(rootEntity, Object3DComponent)) { + const obj3d = new Group() + setComponent(rootEntity, Object3DComponent, obj3d) + addObjectToGroup(rootEntity, obj3d) + proxifyParentChildRelationships(obj3d) + } + + console.log(gltf) + + if (gltf.extensions?.VRM) { + const vrmExtensionDefinition = gltf.extensions!.VRM as V0VRM.VRM + const bones = vrmExtensionDefinition.humanoid!.humanBones!.reduce((bones, bone) => { + const nodeID = `${documentID}-${bone.node}` as EntityUUID + const entity = UUIDComponent.getEntityByUUID(nodeID) + bones[bone.bone!] = { node: getComponent(entity, BoneComponent) } + return bones + }, {} as VRMHumanBones) + console.log({ bones }) + + bones.hips.node.rotateY(Math.PI) + + const humanoid = new VRMHumanoid(bones) + console.log({ humanoid }) + const scene = getComponent(rootEntity, Object3DComponent) as any as Group + + const meta = vrmExtensionDefinition.meta! as any + + const vrm = new VRM({ + scene, + humanoid, + meta + // expressionManager: gltf.userData.vrmExpressionManager, + // firstPerson: gltf.userData.vrmFirstPerson, + // lookAt: gltf.userData.vrmLookAt, + // materials: gltf.userData.vrmMToonMaterials, + // springBoneManager: gltf.userData.vrmSpringBoneManager, + // nodeConstraintManager: gltf.userData.vrmNodeConstraintManager, + }) + + vrm.humanoid.normalizedHumanBonesRoot.removeFromParent() + console.log(vrm) + + return vrm + } + + return createVRMFromGLTF(rootEntity, gltf) +} + +const hipsRegex = /hip|pelvis/i +export const recursiveHipsLookup = (entity: Entity): Entity | undefined => { + const name = getComponent(entity, NameComponent).toLowerCase() + if (hipsRegex.test(name)) { + return entity + } + const children = getComponent(entity, EntityTreeComponent).children + for (const child of children) { + const e = recursiveHipsLookup(child) + if (e) return e + } +} + +const createVRMFromGLTF = (rootEntity: Entity, gltf: GLTF.IGLTF) => { + const hipsEntity = recursiveHipsLookup(rootEntity)! + + const hipsName = getComponent(hipsEntity, NameComponent) + + const bones = {} as VRMHumanBones + + /** + * some mixamo rigs do not use the mixamo prefix, if so we add + * a prefix to the rig names for matching to keys in the mixamoVRMRigMap + */ + const mixamoPrefix = hipsName.includes('mixamorig') ? '' : 'mixamorig' + /** + * some mixamo rigs have an identifier or suffix after the mixamo prefix + * that must be removed for matching to keys in the mixamoVRMRigMap + */ + const removeSuffix = mixamoPrefix ? false : !/[hp]/i.test(hipsName.charAt(9)) + console.log({ removeSuffix, mixamoPrefix }) + + iterateEntityNode(hipsEntity, (entity) => { + // if (!getComponent(entity, BoneComponent)) return + + const name = getComponent(entity, NameComponent) + /**match the keys to create a humanoid bones object */ + let boneName = mixamoPrefix + name + if (removeSuffix) boneName = boneName.slice(0, 9) + name.slice(10) + const bone = mixamoVRMRigMap[boneName] as string + console.log({ name, boneName, removeSuffix, bone, mixamoVRMRigMap }) + if (bone) { + bones[bone] = { node: getComponent(entity, BoneComponent) } as VRMHumanBone + } + }) + + console.log(bones) + const humanoid = enforceTPose(new VRMHumanoid(bones)) + const scene = getComponent(rootEntity, Object3DComponent) + const children = getComponent(rootEntity, EntityTreeComponent).children + const childName = getComponent(children[0], NameComponent) + + const vrm = new VRM({ + humanoid, + scene: scene, + meta: { name: childName } as VRM1Meta + // expressionManager: gltf.userData.vrmExpressionManager, + // firstPerson: gltf.userData.vrmFirstPerson, + // lookAt: gltf.userData.vrmLookAt, + // materials: gltf.userData.vrmMToonMaterials, + // springBoneManager: gltf.userData.vrmSpringBoneManager, + // nodeConstraintManager: gltf.userData.vrmNodeConstraintManager, + } as VRMParameters) + + if (!vrm.userData) vrm.userData = {} + humanoid.humanBones.rightHand.node.getWorldPosition(_rightHandPos) + humanoid.humanBones.rightUpperArm.node.getWorldPosition(_rightUpperArmPos) + return vrm +} + +const legAngle = new Euler(0, 0, Math.PI) +const rightShoulderAngle = new Euler(Math.PI / 2, 0, Math.PI / 2) +const leftShoulderAngle = new Euler(Math.PI / 2, 0, -Math.PI / 2) +export const enforceTPose = (humanoid: VRMHumanoid) => { + const bones = humanoid.humanBones + console.log('enforcing T pose', humanoid, bones) + + bones.rightShoulder!.node.quaternion.setFromEuler(rightShoulderAngle) + bones.rightUpperArm.node.quaternion.set(0, 0, 0, 1) + bones.rightLowerArm.node.quaternion.set(0, 0, 0, 1) + + bones.leftShoulder!.node.quaternion.setFromEuler(leftShoulderAngle) + bones.leftUpperArm.node.quaternion.set(0, 0, 0, 1) + bones.leftLowerArm.node.quaternion.set(0, 0, 0, 1) + + bones.rightUpperLeg.node.quaternion.setFromEuler(legAngle) + bones.rightLowerLeg.node.quaternion.set(0, 0, 0, 1) + + bones.leftUpperLeg.node.quaternion.setFromEuler(legAngle) + bones.leftLowerLeg.node.quaternion.set(0, 0, 0, 1) + + return new VRMHumanoid(bones) +} diff --git a/packages/engine/src/avatar/state/AvatarNetworkState.tsx b/packages/engine/src/avatar/state/AvatarNetworkState.tsx index a02a8be9b5..d02b2f3b3a 100644 --- a/packages/engine/src/avatar/state/AvatarNetworkState.tsx +++ b/packages/engine/src/avatar/state/AvatarNetworkState.tsx @@ -42,8 +42,8 @@ import { import { WorldNetworkAction } from '@etherealengine/network' import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' +import { AvatarRigComponent } from '../components/AvatarAnimationComponent' import { AvatarColliderComponent } from '../components/AvatarControllerComponent' -import { loadAvatarModelAsset, unloadAvatarForUser } from '../functions/avatarFunctions' import { spawnAvatarReceptor } from '../functions/spawnAvatarReceptor' import { AvatarNetworkAction } from './AvatarNetworkActions' @@ -142,11 +142,11 @@ const AvatarReactor = ({ entityUUID }: { entityUUID: EntityUUID }) => { if (!isClient) return if (!entity || !userAvatarDetails.value) return - loadAvatarModelAsset(entity, userAvatarDetails.value) + setComponent(entity, AvatarRigComponent, { avatarURL: userAvatarDetails.value }) return () => { if (!entityExists(entity)) return - unloadAvatarForUser(entity) + setComponent(entity, AvatarRigComponent, { avatarURL: '' }) } }, [userAvatarDetails, entity]) diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 6aa73bc750..cf7a89d76f 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -625,7 +625,7 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: const entityState = useHookstate(UndefinedEntity) const entity = entityState.value - useEffect(() => { + useLayoutEffect(() => { const uuid = getNodeUUID(node.get(NO_PROXY) as GLTF.IGLTF, props.documentID, props.nodeIndex) const entity = UUIDComponent.getOrCreateEntityByUUID(uuid) @@ -1003,6 +1003,7 @@ const PrimitiveReactor = (props: { /** @todo multiple primitive support */ addObjectToGroup(entity, mesh) + proxifyParentChildRelationships(mesh) return () => { removeComponent(entity, SkinnedMeshComponent) From 8e42916177c589a9127e0e50b8395f9ff8837096 Mon Sep 17 00:00:00 2001 From: AidanCaruso Date: Wed, 14 Aug 2024 12:58:44 -0400 Subject: [PATCH 37/47] wip fix avatar retargeting --- .../engine/src/avatar/components/AvatarAnimationComponent.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index 12e6783c2f..4e457a2186 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -261,6 +261,7 @@ export default function createVRM(rootEntity: Entity) { const humanoid = new VRMHumanoid(bones) console.log({ humanoid }) const scene = getComponent(rootEntity, Object3DComponent) as any as Group + scene.rotation.y = Math.PI const meta = vrmExtensionDefinition.meta! as any From 2c51a398d462667a4d2be1981222c2abf3f1953e Mon Sep 17 00:00:00 2001 From: AidanCaruso Date: Wed, 14 Aug 2024 14:22:10 -0400 Subject: [PATCH 38/47] conditional flip only if vrm 0 --- .../engine/src/avatar/components/AvatarAnimationComponent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index 4e457a2186..c2fe51279b 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -261,7 +261,7 @@ export default function createVRM(rootEntity: Entity) { const humanoid = new VRMHumanoid(bones) console.log({ humanoid }) const scene = getComponent(rootEntity, Object3DComponent) as any as Group - scene.rotation.y = Math.PI + scene.rotation.y = vrmExtensionDefinition.meta?.version === '0' ? Math.PI : 0 const meta = vrmExtensionDefinition.meta! as any From d5d735e0c81f9cdc02b9b2a82884188299e57b3d Mon Sep 17 00:00:00 2001 From: AidanCaruso Date: Wed, 14 Aug 2024 14:27:54 -0400 Subject: [PATCH 39/47] cleanup --- .../src/avatar/components/AvatarAnimationComponent.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index c2fe51279b..4c394e2057 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -255,13 +255,11 @@ export default function createVRM(rootEntity: Entity) { return bones }, {} as VRMHumanBones) console.log({ bones }) - - bones.hips.node.rotateY(Math.PI) - + if (vrmExtensionDefinition.meta?.version === '0') bones.hips.node.rotateY(Math.PI) const humanoid = new VRMHumanoid(bones) console.log({ humanoid }) + const scene = getComponent(rootEntity, Object3DComponent) as any as Group - scene.rotation.y = vrmExtensionDefinition.meta?.version === '0' ? Math.PI : 0 const meta = vrmExtensionDefinition.meta! as any From 8da8f4529633e534c0a067697458a0eec4795812 Mon Sep 17 00:00:00 2001 From: AidanCaruso Date: Wed, 14 Aug 2024 16:12:37 -0400 Subject: [PATCH 40/47] wip set loaded avatar world matrices to identity and flip --- .../components/AvatarAnimationComponent.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index 4c394e2057..7be94cab79 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -244,20 +244,27 @@ export default function createVRM(rootEntity: Entity) { proxifyParentChildRelationships(obj3d) } - console.log(gltf) - if (gltf.extensions?.VRM) { const vrmExtensionDefinition = gltf.extensions!.VRM as V0VRM.VRM + const flip = new Matrix4().makeRotationFromEuler(new Euler(0, Math.PI, 0)) const bones = vrmExtensionDefinition.humanoid!.humanBones!.reduce((bones, bone) => { const nodeID = `${documentID}-${bone.node}` as EntityUUID const entity = UUIDComponent.getEntityByUUID(nodeID) bones[bone.bone!] = { node: getComponent(entity, BoneComponent) } + console.log(bones[bone.bone!].node) return bones }, {} as VRMHumanBones) - console.log({ bones }) - if (vrmExtensionDefinition.meta?.version === '0') bones.hips.node.rotateY(Math.PI) + + /**hacky, @todo test with vrm1 */ + iterateEntityNode(bones.hips.node.parent!.entity, (entity) => { + const bone = getComponent(entity, BoneComponent) + bone.matrixWorld.identity() + if (bone.entity != bones.hips.node.parent!.entity) bone.matrixWorld.multiply(flip) + }) + bones.hips.node.rotateY(Math.PI) + const humanoid = new VRMHumanoid(bones) - console.log({ humanoid }) + humanoid.normalizedHumanBonesRoot.removeFromParent() const scene = getComponent(rootEntity, Object3DComponent) as any as Group @@ -275,7 +282,6 @@ export default function createVRM(rootEntity: Entity) { // nodeConstraintManager: gltf.userData.vrmNodeConstraintManager, }) - vrm.humanoid.normalizedHumanBonesRoot.removeFromParent() console.log(vrm) return vrm From 0e35c95f01dd76efa203369e0e2fd75c9434dc48 Mon Sep 17 00:00:00 2001 From: HexaField Date: Thu, 15 Aug 2024 11:02:16 +1000 Subject: [PATCH 41/47] add back skeleton helper, move skinned mesh and bone components to spatial module --- .../src/media/webcam/WebcamInput.ts | 2 +- .../components/AvatarAnimationComponent.ts | 49 +---- .../components/AvatarDissolveComponent.ts | 2 +- .../avatar/components/SkinnedMeshComponent.ts | 92 ---------- .../avatar/functions/updateVRMRetargeting.ts | 2 +- .../avatar/systems/AvatarAnimationSystem.tsx | 2 +- .../engine/src/gltf/GLTFLoaderFunctions.ts | 2 +- packages/engine/src/gltf/GLTFState.tsx | 18 +- .../src/scene/functions/loadGLTFModel.ts | 4 +- .../src/renderer}/components/BoneComponent.ts | 0 .../components/SkinnedMeshComponent.ts | 173 ++++++++++++++++++ 11 files changed, 192 insertions(+), 154 deletions(-) delete mode 100644 packages/engine/src/avatar/components/SkinnedMeshComponent.ts rename packages/{engine/src/avatar => spatial/src/renderer}/components/BoneComponent.ts (100%) create mode 100644 packages/spatial/src/renderer/components/SkinnedMeshComponent.ts diff --git a/packages/client-core/src/media/webcam/WebcamInput.ts b/packages/client-core/src/media/webcam/WebcamInput.ts index 2bf02b3719..27563de27e 100755 --- a/packages/client-core/src/media/webcam/WebcamInput.ts +++ b/packages/client-core/src/media/webcam/WebcamInput.ts @@ -8,12 +8,12 @@ import { Entity } from '@etherealengine/ecs/src/Entity' import { defineQuery } from '@etherealengine/ecs/src/QueryFunctions' import { AvatarRigComponent } from '@etherealengine/engine/src/avatar/components/AvatarAnimationComponent' import { AvatarComponent } from '@etherealengine/engine/src/avatar/components/AvatarComponent' -import { SkinnedMeshComponent } from '@etherealengine/engine/src/avatar/components/SkinnedMeshComponent' import { AvatarNetworkAction } from '@etherealengine/engine/src/avatar/state/AvatarNetworkActions' import { defineActionQueue, getMutableState } from '@etherealengine/hyperflux' import { GroupComponent } from '@etherealengine/spatial/src/renderer/components/GroupComponent' import { iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' import { MediaStreamState } from '../../transports/MediaStreams' import { WebcamInputComponent } from './WebcamInputComponent' diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index 7be94cab79..2014745e6f 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -35,7 +35,7 @@ import { import type * as V0VRM from '@pixiv/types-vrm-0.0' import { useEffect } from 'react' -import { AnimationAction, Euler, Group, Matrix4, SkeletonHelper, Vector3 } from 'three' +import { AnimationAction, Euler, Group, Matrix4, Vector3 } from 'three' import { defineComponent, @@ -47,26 +47,23 @@ import { useOptionalComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { Entity, EntityUUID } from '@etherealengine/ecs/src/Entity' -import { createEntity, removeEntity, useEntityContext } from '@etherealengine/ecs/src/EntityFunctions' -import { getMutableState, getState, matches, none, useHookstate } from '@etherealengine/hyperflux' +import { useEntityContext } from '@etherealengine/ecs/src/EntityFunctions' +import { getMutableState, getState, matches, useHookstate } from '@etherealengine/hyperflux' import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' -import { setObjectLayers } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' -import { setVisibleComponent, VisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' -import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' -import { RendererState } from '@etherealengine/spatial/src/renderer/RendererState' import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' import { UUIDComponent } from '@etherealengine/ecs' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' import { EntityTreeComponent, iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' import { GLTF } from '@gltf-transform/core' import { GLTFComponent } from '../../gltf/GLTFComponent' import { GLTFDocumentState } from '../../gltf/GLTFDocumentState' import { proxifyParentChildRelationships } from '../../scene/functions/loadGLTFModel' -import { preloadedAnimations } from '../animation/Util' import { AnimationState } from '../AnimationManager' import { mixamoVRMRigMap } from '../AvatarBoneMatching' +import { preloadedAnimations } from '../animation/Util' import { retargetAvatarAnimations, setAvatarSpeedFromRootMotion, @@ -75,8 +72,6 @@ import { } from '../functions/avatarFunctions' import { AvatarState } from '../state/AvatarNetworkState' import { AvatarComponent } from './AvatarComponent' -import { AvatarPendingComponent } from './AvatarPendingComponent' -import { BoneComponent } from './BoneComponent' export const AvatarAnimationComponent = defineComponent({ name: 'AvatarAnimationComponent', @@ -144,44 +139,12 @@ export const AvatarRigComponent = defineComponent({ reactor: function () { const entity = useEntityContext() - const debugEnabled = useHookstate(getMutableState(RendererState).avatarDebug) const rigComponent = useComponent(entity, AvatarRigComponent) - const pending = useOptionalComponent(entity, AvatarPendingComponent) - const visible = useOptionalComponent(entity, VisibleComponent) const gltfComponent = useOptionalComponent(entity, GLTFComponent) const locomotionAnimationState = useHookstate( getMutableState(AnimationState).loadedAnimations[preloadedAnimations.locomotion] ) - useEffect(() => { - if (!visible?.value || !debugEnabled.value || pending?.value || !rigComponent.value.normalizedRig?.hips?.node) - return - - const helper = new SkeletonHelper(rigComponent.value.vrm.scene as Group) - helper.frustumCulled = false - helper.name = `target-rig-helper-${entity}` - - const helperEntity = createEntity() - setVisibleComponent(helperEntity, true) - addObjectToGroup(helperEntity, helper) - rigComponent.helperEntity.set(helperEntity) - setComponent(helperEntity, NameComponent, helper.name) - setObjectLayers(helper, ObjectLayers.AvatarHelper) - - setComponent(helperEntity, ComputedTransformComponent, { - referenceEntities: [entity], - computeFunction: () => { - // this updates the bone helper lines - helper.updateMatrixWorld(true) - } - }) - - return () => { - removeEntity(helperEntity) - rigComponent.helperEntity.set(none) - } - }, [visible, debugEnabled, pending, rigComponent.normalizedRig]) - /** @todo move asset loading to a new VRMComponent */ useEffect(() => { if (!rigComponent?.avatarURL?.value) return @@ -251,7 +214,6 @@ export default function createVRM(rootEntity: Entity) { const nodeID = `${documentID}-${bone.node}` as EntityUUID const entity = UUIDComponent.getEntityByUUID(nodeID) bones[bone.bone!] = { node: getComponent(entity, BoneComponent) } - console.log(bones[bone.bone!].node) return bones }, {} as VRMHumanBones) @@ -264,7 +226,6 @@ export default function createVRM(rootEntity: Entity) { bones.hips.node.rotateY(Math.PI) const humanoid = new VRMHumanoid(bones) - humanoid.normalizedHumanBonesRoot.removeFromParent() const scene = getComponent(rootEntity, Object3DComponent) as any as Group diff --git a/packages/engine/src/avatar/components/AvatarDissolveComponent.ts b/packages/engine/src/avatar/components/AvatarDissolveComponent.ts index 45a0056334..3ab007a559 100644 --- a/packages/engine/src/avatar/components/AvatarDissolveComponent.ts +++ b/packages/engine/src/avatar/components/AvatarDissolveComponent.ts @@ -48,7 +48,7 @@ import { matches } from '@etherealengine/hyperflux' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' -import { SkinnedMeshComponent } from './SkinnedMeshComponent' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' export type MaterialMap = { entity: Entity diff --git a/packages/engine/src/avatar/components/SkinnedMeshComponent.ts b/packages/engine/src/avatar/components/SkinnedMeshComponent.ts deleted file mode 100644 index f2acaf10fe..0000000000 --- a/packages/engine/src/avatar/components/SkinnedMeshComponent.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* -CPAL-1.0 License - -The contents of this file are subject to the Common Public Attribution License -Version 1.0. (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at -https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. -The License is based on the Mozilla Public License Version 1.1, but Sections 14 -and 15 have been added to cover use of software over a computer network and -provide for limited attribution for the Original Developer. In addition, -Exhibit A has been modified to be consistent with Exhibit B. - -Software distributed under the License is distributed on an "AS IS" basis, -WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the -specific language governing rights and limitations under the License. - -The Original Code is Ethereal Engine. - -The Original Developer is the Initial Developer. The Initial Developer of the -Original Code is the Ethereal Engine team. - -All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 -Ethereal Engine. All Rights Reserved. -*/ - -import { SkeletonHelper, SkinnedMesh } from 'three' - -import { createEntity, removeEntity, useEntityContext } from '@etherealengine/ecs' -import { - defineComponent, - setComponent, - useComponent, - useOptionalComponent -} from '@etherealengine/ecs/src/ComponentFunctions' -import { getMutableState, getState, useHookstate } from '@etherealengine/hyperflux' -import { EngineState } from '@etherealengine/spatial/src/EngineState' -import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' -import { RendererState } from '@etherealengine/spatial/src/renderer/RendererState' -import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' -import { setObjectLayers } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' -import { VisibleComponent, setVisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' -import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' -import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' -import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' -import { useEffect } from 'react' - -/** @todo move this to spatial module */ -export const SkinnedMeshComponent = defineComponent({ - name: 'SkinnedMeshComponent', - - onInit: (entity) => null! as SkinnedMesh, - - onSet: (entity, component, mesh: SkinnedMesh) => { - if (!mesh || !mesh.isSkinnedMesh) throw new Error('SkinnedMeshComponent: Invalid skinned mesh') - component.set(mesh) - }, - - reactor: function () { - const entity = useEntityContext() - const component = useComponent(entity, SkinnedMeshComponent) - const debugEnabled = useHookstate(getMutableState(RendererState).avatarDebug) - const visible = useOptionalComponent(entity, VisibleComponent) - - useEffect(() => { - if (!visible?.value || !debugEnabled.value) return - - const helper = new SkeletonHelper(component.value as SkinnedMesh) - helper.frustumCulled = false - helper.name = `Skinned Mesh Helper For: ${entity}` - - const helperEntity = createEntity() - setVisibleComponent(helperEntity, true) - addObjectToGroup(helperEntity, helper) - setComponent(helperEntity, NameComponent, helper.name) - setObjectLayers(helper, ObjectLayers.AvatarHelper) - setComponent(helperEntity, EntityTreeComponent, { parentEntity: getState(EngineState).originEntity }) - - setComponent(helperEntity, ComputedTransformComponent, { - referenceEntities: [entity], - computeFunction: () => { - // this updates the bone helper lines - helper.updateMatrixWorld(true) - } - }) - return () => { - removeEntity(helperEntity) - } - }, [visible, debugEnabled, component.skeleton.value]) - - return null - } -}) diff --git a/packages/engine/src/avatar/functions/updateVRMRetargeting.ts b/packages/engine/src/avatar/functions/updateVRMRetargeting.ts index a60f9f3c54..5aa9117bfd 100644 --- a/packages/engine/src/avatar/functions/updateVRMRetargeting.ts +++ b/packages/engine/src/avatar/functions/updateVRMRetargeting.ts @@ -31,8 +31,8 @@ import { Entity } from '@etherealengine/ecs/src/Entity' import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' import { TransformComponent } from '@etherealengine/spatial/src/transform/components/TransformComponent' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { AvatarComponent } from '../components/AvatarComponent' -import { BoneComponent } from '../components/BoneComponent' export const updateVRMRetargeting = (vrm: VRM, avatarEntity: Entity) => { const humanoidRig = (vrm.humanoid as any)._normalizedHumanBones // as VRMHumanoidRig diff --git a/packages/engine/src/avatar/systems/AvatarAnimationSystem.tsx b/packages/engine/src/avatar/systems/AvatarAnimationSystem.tsx index 252bcb3411..c72e5f03dd 100644 --- a/packages/engine/src/avatar/systems/AvatarAnimationSystem.tsx +++ b/packages/engine/src/avatar/systems/AvatarAnimationSystem.tsx @@ -59,6 +59,7 @@ import { XRLeftHandComponent, XRRightHandComponent } from '@etherealengine/spati import { XRState } from '@etherealengine/spatial/src/xr/XRState' import { EngineState } from '@etherealengine/spatial/src/EngineState' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' import React from 'react' import { useBatchGLTF } from '../../assets/functions/resourceLoaderHooks' import { GLTF } from '../../assets/loaders/gltf/GLTFLoader' @@ -72,7 +73,6 @@ import { AnimationComponent } from '../components/AnimationComponent' import { AvatarAnimationComponent, AvatarRigComponent } from '../components/AvatarAnimationComponent' import { AvatarComponent } from '../components/AvatarComponent' import { AvatarIKTargetComponent } from '../components/AvatarIKComponents' -import { SkinnedMeshComponent } from '../components/SkinnedMeshComponent' import { retargetAnimationClip } from '../functions/retargetMixamoRig' import { updateVRMRetargeting } from '../functions/updateVRMRetargeting' import { IKSerialization } from '../IKSerialization' diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 31c374e4dc..9953720964 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -33,6 +33,7 @@ import { } from '@etherealengine/ecs' import { NO_PROXY, getState, startReactor, useHookstate } from '@etherealengine/hyperflux' import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { MaterialPrototypeComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTF } from '@gltf-transform/core' @@ -94,7 +95,6 @@ import { import { GLTFParserOptions, GLTFRegistry, getImageURIMimeType } from '../assets/loaders/gltf/GLTFParser' import { KTX2Loader } from '../assets/loaders/gltf/KTX2Loader' import { AssetLoaderState } from '../assets/state/AssetLoaderState' -import { BoneComponent } from '../avatar/components/BoneComponent' import { KHR_DRACO_MESH_COMPRESSION, getBufferIndex } from './GLTFExtensions' import { KHRTextureTransformExtensionComponent, MaterialDefinitionComponent } from './MaterialDefinitionComponent' diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index cf7a89d76f..0b9ae561f0 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -88,13 +88,13 @@ import { import { CameraComponent } from '@etherealengine/spatial/src/camera/components/CameraComponent' import { EngineState } from '@etherealengine/spatial/src/EngineState' import { Physics } from '@etherealengine/spatial/src/physics/classes/Physics' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' import { MaterialInstanceComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTFParserOptions } from '../assets/loaders/gltf/GLTFParser' import { AssetLoaderState } from '../assets/state/AssetLoaderState' import { AnimationComponent } from '../avatar/components/AnimationComponent' -import { BoneComponent } from '../avatar/components/BoneComponent' -import { SkinnedMeshComponent } from '../avatar/components/SkinnedMeshComponent' import { SourceComponent } from '../scene/components/SourceComponent' import { proxifyParentChildRelationships } from '../scene/functions/loadGLTFModel' import { GLTFComponent } from './GLTFComponent' @@ -659,17 +659,13 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: } if (!hasComponent(entity, Object3DComponent) && !hasComponent(entity, MeshComponent)) { - let obj3d: Group | Bone if (isBoneNode(documentState.get(NO_PROXY) as GLTF.IGLTF, props.nodeIndex)) { - obj3d = new Bone() - setComponent(entity, BoneComponent, obj3d) - } else { - obj3d = new Group() + const bone = new Bone() + setComponent(entity, BoneComponent, bone) + addObjectToGroup(entity, bone) + proxifyParentChildRelationships(bone) + setComponent(entity, Object3DComponent, bone) } - obj3d.entity = entity - addObjectToGroup(entity, obj3d) - proxifyParentChildRelationships(obj3d) - setComponent(entity, Object3DComponent, obj3d) } entityState.set(entity) diff --git a/packages/engine/src/scene/functions/loadGLTFModel.ts b/packages/engine/src/scene/functions/loadGLTFModel.ts index eff4b293ee..356bc123fd 100644 --- a/packages/engine/src/scene/functions/loadGLTFModel.ts +++ b/packages/engine/src/scene/functions/loadGLTFModel.ts @@ -50,8 +50,8 @@ import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/compo import { computeTransformMatrix } from '@etherealengine/spatial/src/transform/systems/TransformSystem' import { ColliderComponent } from '@etherealengine/spatial/src/physics/components/ColliderComponent' -import { BoneComponent } from '../../avatar/components/BoneComponent' -import { SkinnedMeshComponent } from '../../avatar/components/SkinnedMeshComponent' +import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' +import { SkinnedMeshComponent } from '@etherealengine/spatial/src/renderer/components/SkinnedMeshComponent' import { GLTFLoadedComponent } from '../components/GLTFLoadedComponent' import { InstancingComponent } from '../components/InstancingComponent' import { ModelComponent } from '../components/ModelComponent' diff --git a/packages/engine/src/avatar/components/BoneComponent.ts b/packages/spatial/src/renderer/components/BoneComponent.ts similarity index 100% rename from packages/engine/src/avatar/components/BoneComponent.ts rename to packages/spatial/src/renderer/components/BoneComponent.ts diff --git a/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts b/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts new file mode 100644 index 0000000000..61888472e5 --- /dev/null +++ b/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts @@ -0,0 +1,173 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + +import { + BufferGeometry, + Color, + Float32BufferAttribute, + LineBasicMaterial, + LineSegments, + Matrix4, + SkinnedMesh, + Vector3 +} from 'three' + +import { createEntity, removeEntity, useEntityContext } from '@etherealengine/ecs' +import { + defineComponent, + getComponent, + getOptionalComponent, + hasComponent, + setComponent, + useComponent, + useOptionalComponent +} from '@etherealengine/ecs/src/ComponentFunctions' +import { getMutableState, useHookstate } from '@etherealengine/hyperflux' +import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' +import { RendererState } from '@etherealengine/spatial/src/renderer/RendererState' +import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' +import { VisibleComponent, setVisibleComponent } from '@etherealengine/spatial/src/renderer/components/VisibleComponent' +import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' +import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { useEffect } from 'react' +import { TransformComponent } from '../RendererModule' +import { ObjectLayers } from '../constants/ObjectLayers' +import { BoneComponent } from './BoneComponent' +import { setObjectLayers } from './ObjectLayerComponent' + +export const SkinnedMeshComponent = defineComponent({ + name: 'SkinnedMeshComponent', + + onInit: (entity) => null! as SkinnedMesh, + + onSet: (entity, component, mesh: SkinnedMesh) => { + if (!mesh || !mesh.isSkinnedMesh) throw new Error('SkinnedMeshComponent: Invalid skinned mesh') + component.set(mesh) + }, + + reactor: function () { + const entity = useEntityContext() + const component = useComponent(entity, SkinnedMeshComponent) + const debugEnabled = useHookstate(getMutableState(RendererState).avatarDebug) + const visible = useOptionalComponent(entity, VisibleComponent) + + useEffect(() => { + if (!visible?.value || !debugEnabled.value) return + + const root = getComponent(entity, SkinnedMeshComponent) + const bones = root.skeleton.bones //getBoneList(entity) + + const geometry = new BufferGeometry() + + const vertices = [] as number[] + const colors = [] as number[] + + const color1 = new Color(0, 0, 1) + const color2 = new Color(0, 1, 0) + + for (let i = 0; i < bones.length; i++) { + const bone = bones[i] + + const boneParentEntity = getComponent(bone.entity, EntityTreeComponent).parentEntity + const boneParentComponent = hasComponent(boneParentEntity, BoneComponent) + + if (boneParentComponent) { + vertices.push(0, 0, 0) + vertices.push(0, 0, 0) + colors.push(color1.r, color1.g, color1.b) + colors.push(color2.r, color2.g, color2.b) + } + } + + geometry.setAttribute('position', new Float32BufferAttribute(vertices, 3)) + geometry.setAttribute('color', new Float32BufferAttribute(colors, 3)) + + const material = new LineBasicMaterial({ + vertexColors: true, + depthTest: false, + depthWrite: false, + toneMapped: false, + transparent: true + }) + + const line = new LineSegments(geometry, material) + + line.frustumCulled = false + line.name = `Skinned Mesh Helper For: ${entity}` + + const helperEntity = createEntity() + setVisibleComponent(helperEntity, true) + addObjectToGroup(helperEntity, line) + setComponent(helperEntity, NameComponent, line.name) + setObjectLayers(line, ObjectLayers.AvatarHelper) + setComponent(helperEntity, EntityTreeComponent, { parentEntity: entity }) + + setComponent(helperEntity, ComputedTransformComponent, { + referenceEntities: [entity], + computeFunction: () => { + const position = geometry.getAttribute('position') + + _matrixWorldInv.copy(root.matrixWorld).invert() + + for (let i = 0, j = 0; i < bones.length; i++) { + const bone = bones[i] + + const boneParentEntity = getComponent(bone.entity, EntityTreeComponent).parentEntity + const boneParentComponent = getOptionalComponent(boneParentEntity, BoneComponent) + + if (boneParentComponent) { + _boneMatrix.multiplyMatrices(_matrixWorldInv, bone.matrixWorld) + _vector.setFromMatrixPosition(_boneMatrix) + position.setXYZ(j, _vector.x, _vector.y, _vector.z) + + _boneMatrix.multiplyMatrices( + _matrixWorldInv, + getComponent(boneParentEntity, TransformComponent).matrixWorld + ) + _vector.setFromMatrixPosition(_boneMatrix) + position.setXYZ(j + 1, _vector.x, _vector.y, _vector.z) + + j += 2 + } + } + + geometry.getAttribute('position').needsUpdate = true + } + }) + + return () => { + removeEntity(helperEntity) + geometry.dispose() + material.dispose() + } + }, [visible, debugEnabled, component.skeleton.value]) + + return null + } +}) + +const _vector = /*@__PURE__*/ new Vector3() +const _boneMatrix = /*@__PURE__*/ new Matrix4() +const _matrixWorldInv = /*@__PURE__*/ new Matrix4() From ef0e1f6beeb0835d56eea6112db05eecf14deb30 Mon Sep 17 00:00:00 2001 From: HexaField Date: Thu, 15 Aug 2024 11:49:45 +1000 Subject: [PATCH 42/47] material bug fix --- packages/engine/src/gltf/GLTFLoaderFunctions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 9953720964..f01f3ec38b 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -556,10 +556,10 @@ const useLoadMaterial = ( useEffect(() => { const alphaMode = materialDef.alphaMode || ALPHA_MODES.OPAQUE - result.value?.setValues({ transparent: alphaMode !== ALPHA_MODES.OPAQUE }) + result.value?.setValues({ transparent: alphaMode === ALPHA_MODES.BLEND }) // See: https://github.com/mrdoob/three.js/issues/17706 - if (alphaMode !== ALPHA_MODES.OPAQUE) { + if (alphaMode === ALPHA_MODES.BLEND) { result.value?.setValues({ depthWrite: false }) } if (material) material.needsUpdate = true From 3e60296afd45e11a830518214edac1dfd55e4825 Mon Sep 17 00:00:00 2001 From: HexaField Date: Thu, 15 Aug 2024 19:42:20 +1000 Subject: [PATCH 43/47] Replace some usages of ModelComponent with GLTFComponent, and start to support LoopAnimationComponent --- .../common/services/FileThumbnailJobState.tsx | 4 +- .../hierarchy/HierarchyTreeWalker.ts | 4 +- .../src/systems/ClickPlacementSystem.tsx | 3 +- .../gltf/extensions/SourceHandlerExtension.ts | 4 +- .../components/AvatarAnimationComponent.ts | 7 +- .../components/LoopAnimationComponent.ts | 29 +++---- .../src/avatar/components/VRMComponent.ts | 15 ++++ .../src/avatar/systems/AnimationSystem.ts | 11 +-- .../systems/AvatarTransparencySystem.tsx | 4 +- packages/engine/src/gltf/GLTFComponent.tsx | 34 ++++++++ .../engine/src/gltf/GLTFLoaderFunctions.ts | 18 +++-- packages/engine/src/gltf/GLTFState.tsx | 18 ++++- .../src/scene/components/ModelComponent.tsx | 56 ++------------ .../components/ObjectGridSnapComponent.ts | 9 ++- .../scene/functions/loaders/ModelFunctions.ts | 14 +--- .../scene/functions/migrateOldColliders.ts | 4 +- .../functions/materialSourcingFunctions.ts | 4 +- .../src/scene/systems/SceneObjectSystem.tsx | 13 ++-- .../engine/src/scene/systems/ShadowSystem.tsx | 2 +- .../components/SkinnedMeshComponent.ts | 2 +- packages/spatial/src/threejsPatches.ts | 77 ++++++++++++++++++- .../src/transform/components/EntityTree.tsx | 75 ++++++++++++++++++ 22 files changed, 279 insertions(+), 128 deletions(-) create mode 100644 packages/engine/src/avatar/components/VRMComponent.ts diff --git a/packages/client-core/src/common/services/FileThumbnailJobState.tsx b/packages/client-core/src/common/services/FileThumbnailJobState.tsx index 96f65cb962..d62ae0bc89 100644 --- a/packages/client-core/src/common/services/FileThumbnailJobState.tsx +++ b/packages/client-core/src/common/services/FileThumbnailJobState.tsx @@ -42,7 +42,6 @@ import { import { useTexture } from '@etherealengine/engine/src/assets/functions/resourceLoaderHooks' import { GLTFDocumentState } from '@etherealengine/engine/src/gltf/GLTFDocumentState' import { ModelComponent } from '@etherealengine/engine/src/scene/components/ModelComponent' -import { getModelSceneID } from '@etherealengine/engine/src/scene/functions/loaders/ModelFunctions' import { NO_PROXY, defineState, getMutableState, useHookstate } from '@etherealengine/hyperflux' import { DirectionalLightComponent, TransformComponent } from '@etherealengine/spatial' import { CameraComponent } from '@etherealengine/spatial/src/camera/components/CameraComponent' @@ -66,6 +65,7 @@ import React, { useEffect } from 'react' import { Color, Euler, Material, MathUtils, Matrix4, Mesh, Quaternion, Sphere, SphereGeometry, Vector3 } from 'three' import config from '@etherealengine/common/src/config' +import { GLTFComponent } from '@etherealengine/engine/src/gltf/GLTFComponent' import { ErrorComponent } from '@etherealengine/engine/src/scene/components/ErrorComponent' import { ShadowComponent } from '@etherealengine/engine/src/scene/components/ShadowComponent' import { useFind } from '@etherealengine/spatial/src/common/functions/FeathersHooks' @@ -414,7 +414,7 @@ const ThumbnailJobReactor = () => { const modelEntity = state.modelEntity.value const lightEntity = state.lightEntity.value - const sceneID = getModelSceneID(modelEntity) + const sceneID = GLTFComponent.getInstanceID(modelEntity) if (!sceneState.value[sceneID]) return try { diff --git a/packages/editor/src/components/hierarchy/HierarchyTreeWalker.ts b/packages/editor/src/components/hierarchy/HierarchyTreeWalker.ts index 29b0d85a9d..21c45d6bc8 100644 --- a/packages/editor/src/components/hierarchy/HierarchyTreeWalker.ts +++ b/packages/editor/src/components/hierarchy/HierarchyTreeWalker.ts @@ -31,9 +31,9 @@ import { getState } from '@etherealengine/hyperflux' import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' import { UUIDComponent } from '@etherealengine/ecs' +import { GLTFComponent } from '@etherealengine/engine/src/gltf/GLTFComponent' import { GLTFSnapshotState } from '@etherealengine/engine/src/gltf/GLTFState' import { ModelComponent } from '@etherealengine/engine/src/scene/components/ModelComponent' -import { getModelSceneID } from '@etherealengine/engine/src/scene/functions/loaders/ModelFunctions' import { GLTF } from '@gltf-transform/core' import { EditorState } from '../../services/EditorServices' @@ -84,7 +84,7 @@ function buildHierarchyTree( array.push(item) if (hasComponent(entity, ModelComponent) && showModelChildren) { - const modelSceneID = getModelSceneID(entity) + const modelSceneID = GLTFComponent.getInstanceID(entity) const snapshotState = getState(GLTFSnapshotState) const snapshots = snapshotState[modelSceneID] if (snapshots) { diff --git a/packages/editor/src/systems/ClickPlacementSystem.tsx b/packages/editor/src/systems/ClickPlacementSystem.tsx index 475a25b9a5..e16c1c9da4 100644 --- a/packages/editor/src/systems/ClickPlacementSystem.tsx +++ b/packages/editor/src/systems/ClickPlacementSystem.tsx @@ -47,7 +47,6 @@ import { ModelComponent } from '@etherealengine/engine/src/scene/components/Mode import { SourceComponent } from '@etherealengine/engine/src/scene/components/SourceComponent' import { entityJSONToGLTFNode } from '@etherealengine/engine/src/scene/functions/GLTFConversion' import { createSceneEntity } from '@etherealengine/engine/src/scene/functions/createSceneEntity' -import { getModelSceneID } from '@etherealengine/engine/src/scene/functions/loaders/ModelFunctions' import { toEntityJson } from '@etherealengine/engine/src/scene/functions/serializeWorld' import { NO_PROXY, @@ -173,7 +172,7 @@ const PlacementModelReactor = (props: { placementEntity: Entity }) => { useEffect(() => { if (!placementModel) return - const sceneID = getModelSceneID(props.placementEntity) + const sceneID = GLTFComponent.getInstanceID(props.placementEntity) if (!sceneState.scenes[sceneID]) return iterateEntityNode(props.placementEntity, (entity) => { const mesh = getOptionalComponent(entity, MeshComponent) diff --git a/packages/engine/src/assets/exporters/gltf/extensions/SourceHandlerExtension.ts b/packages/engine/src/assets/exporters/gltf/extensions/SourceHandlerExtension.ts index f8ab12b5d0..cfc0e70493 100644 --- a/packages/engine/src/assets/exporters/gltf/extensions/SourceHandlerExtension.ts +++ b/packages/engine/src/assets/exporters/gltf/extensions/SourceHandlerExtension.ts @@ -29,8 +29,8 @@ import { getComponent, setComponent } from '@etherealengine/ecs/src/ComponentFun import { Entity, UndefinedEntity } from '@etherealengine/ecs/src/Entity' import { EntityTreeComponent, iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' +import { GLTFComponent } from '../../../../gltf/GLTFComponent' import { SourceComponent } from '../../../../scene/components/SourceComponent' -import { getModelSceneID } from '../../../../scene/functions/loaders/ModelFunctions' import { GLTFExporterPlugin, GLTFWriter } from '../GLTFExporter' import { ExporterExtension } from './ExporterExtension' @@ -46,7 +46,7 @@ export default class SourceHandlerExtension extends ExporterExtension implements //we allow saving of any object that has a source equal to or parent of the root's source const validSrcs: Set = new Set() if (!this.writer.options.srcEntity) return - validSrcs.add(getModelSceneID(this.writer.options.srcEntity!)) + validSrcs.add(GLTFComponent.getInstanceID(this.writer.options.srcEntity!)) const root = (Array.isArray(input) ? input[0] : input) as Object3D let walker: Entity | null = root.entity while (walker !== null) { diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index 2014745e6f..a2960a2cfa 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -72,6 +72,7 @@ import { } from '../functions/avatarFunctions' import { AvatarState } from '../state/AvatarNetworkState' import { AvatarComponent } from './AvatarComponent' +import { VRMComponent } from './VRMComponent' export const AvatarAnimationComponent = defineComponent({ name: 'AvatarAnimationComponent', @@ -163,7 +164,11 @@ export const AvatarRigComponent = defineComponent({ useEffect(() => { if (!rigComponent?.vrm?.value) return const rig = getComponent(entity, AvatarRigComponent) - setupAvatarProportions(entity, rig.vrm as VRM) + setComponent(entity, VRMComponent, rig.vrm) + setupAvatarProportions(entity, rig.vrm) + return () => { + removeComponent(entity, VRMComponent) + } }, [rigComponent?.vrm?.value]) useEffect(() => { diff --git a/packages/engine/src/avatar/components/LoopAnimationComponent.ts b/packages/engine/src/avatar/components/LoopAnimationComponent.ts index 9c69fe25ae..04ca376fd9 100644 --- a/packages/engine/src/avatar/components/LoopAnimationComponent.ts +++ b/packages/engine/src/avatar/components/LoopAnimationComponent.ts @@ -23,7 +23,6 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { VRM } from '@pixiv/three-vrm' import { useEffect } from 'react' import { AnimationAction, @@ -47,9 +46,10 @@ import { NO_PROXY, useHookstate } from '@etherealengine/hyperflux' import { CallbackComponent, StandardCallbacks, setCallback } from '@etherealengine/spatial/src/common/CallbackComponent' import { useGLTF } from '../../assets/functions/resourceLoaderHooks' -import { ModelComponent } from '../../scene/components/ModelComponent' +import { GLTFComponent } from '../../gltf/GLTFComponent' import { bindAnimationClipFromMixamo, retargetAnimationClip } from '../functions/retargetMixamoRig' import { AnimationComponent } from './AnimationComponent' +import { VRMComponent } from './VRMComponent' export const LoopAnimationComponent = defineComponent({ name: 'LoopAnimationComponent', @@ -116,7 +116,8 @@ export const LoopAnimationComponent = defineComponent({ const entity = useEntityContext() const loopAnimationComponent = useComponent(entity, LoopAnimationComponent) - const modelComponent = useOptionalComponent(entity, ModelComponent) + const vrmComponent = useOptionalComponent(entity, VRMComponent) + const gltfComponent = useOptionalComponent(entity, GLTFComponent) const animComponent = useOptionalComponent(entity, AnimationComponent) const animationAction = loopAnimationComponent._action.value as AnimationAction @@ -124,16 +125,14 @@ export const LoopAnimationComponent = defineComponent({ useEffect(() => { if (!animComponent?.animations?.value) return const clip = animComponent.animations.value[loopAnimationComponent.activeClipIndex.value] as AnimationClip - const asset = modelComponent?.asset.get(NO_PROXY) ?? null - if (!modelComponent || !asset?.scene || !clip) { + if (!clip) { loopAnimationComponent._action.set(null) return } animComponent.mixer.time.set(0) - const assetObject = modelComponent.asset.get(NO_PROXY) try { const action = animComponent.mixer.value.clipAction( - assetObject instanceof VRM ? bindAnimationClipFromMixamo(clip, assetObject) : clip + vrmComponent ? bindAnimationClipFromMixamo(clip, getComponent(entity, VRMComponent)) : clip ) loopAnimationComponent._action.set(action) return () => { @@ -142,7 +141,7 @@ export const LoopAnimationComponent = defineComponent({ } catch (e) { console.warn('Failed to bind animation in LoopAnimationComponent', entity, e) } - }, [loopAnimationComponent.activeClipIndex, modelComponent?.asset, animComponent?.animations]) + }, [loopAnimationComponent.activeClipIndex, vrmComponent, animComponent?.animations]) useEffect(() => { if (animationAction?.isRunning()) { @@ -197,23 +196,13 @@ export const LoopAnimationComponent = defineComponent({ setCallback(entity, StandardCallbacks.PAUSE, pause) }, []) - /** - * A model is required for LoopAnimationComponent. - */ - useEffect(() => { - const asset = modelComponent?.asset.get(NO_PROXY) ?? null - if (!asset?.scene) return - const model = getComponent(entity, ModelComponent) - }, [modelComponent?.asset]) - const [gltf] = useGLTF(loopAnimationComponent.animationPack.value, entity) useEffect(() => { - const asset = modelComponent?.asset.get(NO_PROXY) ?? null if ( !gltf || !animComponent || - !asset?.scene || + gltfComponent?.progress.value !== 100 || !loopAnimationComponent.animationPack.value || lastAnimationPack.value === loopAnimationComponent.animationPack.value ) @@ -225,7 +214,7 @@ export const LoopAnimationComponent = defineComponent({ for (let i = 0; i < animations.length; i++) retargetAnimationClip(animations[i], gltf.scene) lastAnimationPack.set(loopAnimationComponent.animationPack.get(NO_PROXY)) animComponent.animations.set(animations) - }, [gltf, animComponent, modelComponent?.asset]) + }, [gltf, animComponent, gltfComponent?.progress]) return null } diff --git a/packages/engine/src/avatar/components/VRMComponent.ts b/packages/engine/src/avatar/components/VRMComponent.ts new file mode 100644 index 0000000000..c1424c5edd --- /dev/null +++ b/packages/engine/src/avatar/components/VRMComponent.ts @@ -0,0 +1,15 @@ +import { defineComponent } from '@etherealengine/ecs' +import { VRM } from '@pixiv/three-vrm' + +export const VRMComponent = defineComponent({ + name: 'VRMComponent', + + onInit(entity) { + return null! as VRM + }, + + onSet(entity, component, json) { + if (!(json instanceof VRM)) throw new Error('Invalid VRM') + component.set(json as VRM) + } +}) diff --git a/packages/engine/src/avatar/systems/AnimationSystem.ts b/packages/engine/src/avatar/systems/AnimationSystem.ts index 69f88a66ac..98f9cddcf0 100644 --- a/packages/engine/src/avatar/systems/AnimationSystem.ts +++ b/packages/engine/src/avatar/systems/AnimationSystem.ts @@ -23,8 +23,6 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { VRM } from '@pixiv/three-vrm' - import { getComponent, getOptionalMutableComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { ECSState } from '@etherealengine/ecs/src/ECSState' import { defineQuery } from '@etherealengine/ecs/src/QueryFunctions' @@ -35,14 +33,14 @@ import { TransformComponent } from '@etherealengine/spatial/src/transform/compon import { TweenComponent } from '@etherealengine/spatial/src/transform/components/TweenComponent' import { TransformDirtyUpdateSystem } from '@etherealengine/spatial/src/transform/systems/TransformSystem' -import { ModelComponent } from '../../scene/components/ModelComponent' import { AnimationComponent } from '.././components/AnimationComponent' import { LoopAnimationComponent } from '../components/LoopAnimationComponent' +import { VRMComponent } from '../components/VRMComponent' import { updateVRMRetargeting } from '../functions/updateVRMRetargeting' const tweenQuery = defineQuery([TweenComponent]) const animationQuery = defineQuery([AnimationComponent, VisibleComponent]) -const loopAnimationQuery = defineQuery([AnimationComponent, LoopAnimationComponent, ModelComponent, TransformComponent]) +const loopAnimationQuery = defineQuery([AnimationComponent, LoopAnimationComponent, VRMComponent, TransformComponent]) const execute = () => { const { deltaSeconds } = getState(ECSState) @@ -62,10 +60,7 @@ const execute = () => { } for (const entity of loopAnimationQuery()) { - const model = getComponent(entity, ModelComponent) - if (model.asset instanceof VRM) { - updateVRMRetargeting(model.asset, entity) - } + updateVRMRetargeting(getComponent(entity, VRMComponent), entity) } } diff --git a/packages/engine/src/avatar/systems/AvatarTransparencySystem.tsx b/packages/engine/src/avatar/systems/AvatarTransparencySystem.tsx index 0222531bd1..a5d00464a2 100644 --- a/packages/engine/src/avatar/systems/AvatarTransparencySystem.tsx +++ b/packages/engine/src/avatar/systems/AvatarTransparencySystem.tsx @@ -50,8 +50,8 @@ import { ditherCalculationType } from '@etherealengine/spatial/src/renderer/materials/constants/plugins/TransparencyDitheringComponent' import React, { useEffect } from 'react' +import { GLTFComponent } from '../../gltf/GLTFComponent' import { SourceComponent } from '../../scene/components/SourceComponent' -import { useModelSceneID } from '../../scene/functions/loaders/ModelFunctions' import { AvatarComponent } from '../components/AvatarComponent' const headDithering = 0 @@ -115,7 +115,7 @@ export const AvatarTransparencySystem = defineSystem({ const AvatarReactor = (props: { entity: Entity }) => { const entity = props.entity - const sceneInstanceID = useModelSceneID(entity) + const sceneInstanceID = GLTFComponent.useInstanceID(entity) const childEntities = useHookstate(SourceComponent.entitiesBySourceState[sceneInstanceID]) return ( <> diff --git a/packages/engine/src/gltf/GLTFComponent.tsx b/packages/engine/src/gltf/GLTFComponent.tsx index 06925656fe..74141d08ad 100644 --- a/packages/engine/src/gltf/GLTFComponent.tsx +++ b/packages/engine/src/gltf/GLTFComponent.tsx @@ -36,6 +36,7 @@ import { hasComponent, useComponent, useEntityContext, + useOptionalComponent, useQuery, UUIDComponent } from '@etherealengine/ecs' @@ -48,6 +49,11 @@ import { useMutableState } from '@etherealengine/hyperflux' +import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' +import { ObjectLayerMaskComponent } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' +import { SceneComponent } from '@etherealengine/spatial/src/renderer/components/SceneComponents' +import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' +import { useAncestorWithComponents } from '@etherealengine/spatial/src/transform/components/EntityTree' import { FileLoader } from '../assets/loaders/base/FileLoader' import { BINARY_EXTENSION_CHUNK_TYPES, @@ -68,6 +74,8 @@ export const GLTFComponent = defineComponent({ onInit(entity) { return { src: '', + /** @todo move this to it's own component */ + cameraOcclusion: false, // internals body: null as null | ArrayBuffer, progress: 0 @@ -76,12 +84,19 @@ export const GLTFComponent = defineComponent({ onSet(entity, component, json) { if (typeof json?.src === 'string') component.src.set(json.src) + if (typeof json?.cameraOcclusion === 'boolean') component.cameraOcclusion.set(json.cameraOcclusion) }, reactor: () => { const entity = useEntityContext() const gltfComponent = useComponent(entity, GLTFComponent) + useEffect(() => { + const occlusion = gltfComponent.cameraOcclusion.value + if (!occlusion) ObjectLayerMaskComponent.disableLayer(entity, ObjectLayers.Camera) + else ObjectLayerMaskComponent.enableLayer(entity, ObjectLayers.Camera) + }, [gltfComponent.cameraOcclusion]) + useGLTFDocument(gltfComponent.src.value, entity) const sourceID = GLTFComponent.getInstanceID(entity) @@ -98,6 +113,13 @@ export const GLTFComponent = defineComponent({ getInstanceID: (entity) => { return `${getComponent(entity, UUIDComponent)}-${getComponent(entity, GLTFComponent).src}` + }, + + useInstanceID: (entity) => { + const uuid = useComponent(entity, UUIDComponent)?.value + const src = useComponent(entity, GLTFComponent)?.src.value + if (!uuid || !src) return '' + return `${uuid}-${src}` } }) @@ -281,3 +303,15 @@ export const parseBinaryData = (data) => { return { json: JSON.parse(content), body } } + +/** + * Returns true if the entity is part of a model or a mesh component that is not a child of model + * @param entity + * @returns {boolean} + */ +export const useHasModelOrIndependentMesh = (entity: Entity) => { + const hasModel = !!useOptionalComponent(entity, GLTFComponent) + const isChildOfModel = !!useAncestorWithComponents(entity, [GLTFComponent, SceneComponent]) + const hasMesh = !!useOptionalComponent(entity, MeshComponent) + return hasModel || (hasMesh && !isChildOfModel) +} diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index f01f3ec38b..0aee5e3ad2 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -35,6 +35,7 @@ import { NO_PROXY, getState, startReactor, useHookstate } from '@etherealengine/ import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' +import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' import { MaterialPrototypeComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTF } from '@gltf-transform/core' import { useEffect } from 'react' @@ -63,6 +64,7 @@ import { MeshBasicMaterial, MeshStandardMaterial, NumberKeyframeTrack, + Object3D, QuaternionKeyframeTrack, RepeatWrapping, SRGBColorSpace, @@ -957,7 +959,7 @@ const useLoadAnimation = (options: GLTFParserOptions, animationIndex?: number) = channels.map((channel, i) => [ i, { - nodes: null as null | Mesh | Bone, + nodes: null as null | Mesh | Bone | Object3D, inputAccessors: null as null | BufferAttribute, outputAccessors: null as null | BufferAttribute, samplers: animationDef.samplers[channel.sampler], @@ -984,15 +986,17 @@ const useLoadAnimation = (options: GLTFParserOptions, animationIndex?: number) = /** @todo we should probably jsut use GroupComponent or something here once we stop creating Object3Ds for all nodes */ const meshComponent = useOptionalComponent(targetNodeEntity, MeshComponent) const boneComponent = useOptionalComponent(targetNodeEntity, BoneComponent) + const obj3dComponent = useOptionalComponent(targetNodeEntity, Object3DComponent) useEffect(() => { const meshWeightsLoaded = meshHasWeights ? meshComponent?.get(NO_PROXY)?.morphTargetInfluences !== undefined : true - if (!meshWeightsLoaded && !boneComponent) return + if (!meshWeightsLoaded && !boneComponent && !obj3dComponent) return channelData[i].nodes.set( getOptionalComponent(targetNodeEntity, MeshComponent) ?? - getOptionalComponent(targetNodeEntity, BoneComponent)! + getOptionalComponent(targetNodeEntity, BoneComponent) ?? + getOptionalComponent(targetNodeEntity, Object3DComponent)! ) }, [meshComponent, boneComponent]) @@ -1014,7 +1018,7 @@ const useLoadAnimation = (options: GLTFParserOptions, animationIndex?: number) = useEffect(() => { if ( Object.values(channelData.get(NO_PROXY)).some( - (data) => data.nodes === null || data.inputAccessors === null || data.outputAccessors === null + (data) => !data.nodes || !data.inputAccessors || !data.outputAccessors ) ) return @@ -1074,13 +1078,15 @@ const _createAnimationTracks = ( ) => { const tracks = [] as any[] // todo - const targetName = node.name ? node.name : node.uuid + const targetName = node.name + if (!targetName) throw new Error('THREE.GLTFLoader: Node has no name.') const targetNames = [] as string[] if (PATH_PROPERTIES[target.path] === PATH_PROPERTIES.weights) { node.traverse(function (object: Mesh | SkinnedMesh) { if (object.morphTargetInfluences) { - targetNames.push(object.name ? object.name : object.uuid) + if (!object.name) throw new Error('THREE.GLTFLoader: Node has no name.') + targetNames.push(object.name) } }) } else { diff --git a/packages/engine/src/gltf/GLTFState.tsx b/packages/engine/src/gltf/GLTFState.tsx index 0b9ae561f0..e897e37272 100644 --- a/packages/engine/src/gltf/GLTFState.tsx +++ b/packages/engine/src/gltf/GLTFState.tsx @@ -424,7 +424,16 @@ export const DocumentReactor = (props: { documentID: string; parentUUID: EntityU const rootEntity = UUIDComponent.useEntityByUUID(props.parentUUID) useEffect(() => { - if (animationState.length !== documentState.value.animations?.length) return + if (animationState.length !== documentState.value?.animations?.length) return + + const hasObject3d = hasComponent(rootEntity, Object3DComponent) + if (!hasObject3d) { + /** @todo this is a temporary hack */ + const obj3d = new Group() + setComponent(rootEntity, Object3DComponent, obj3d) + addObjectToGroup(rootEntity, obj3d) + proxifyParentChildRelationships(obj3d) + } const scene = getComponent(rootEntity, Object3DComponent) scene.animations = animationState.get(NO_PROXY) as AnimationClip[] @@ -435,6 +444,10 @@ export const DocumentReactor = (props: { documentID: string; parentUUID: EntityU }) return () => { removeComponent(rootEntity, AnimationComponent) + if (!hasObject3d) { + removeObjectFromGroup(rootEntity, getComponent(rootEntity, Object3DComponent)) + removeComponent(rootEntity, Object3DComponent) + } } }, [animationState]) @@ -661,6 +674,7 @@ const NodeReactor = (props: { nodeIndex: number; childIndex: number; parentUUID: if (!hasComponent(entity, Object3DComponent) && !hasComponent(entity, MeshComponent)) { if (isBoneNode(documentState.get(NO_PROXY) as GLTF.IGLTF, props.nodeIndex)) { const bone = new Bone() + bone.name = node.name.value ?? 'Bone-' + props.nodeIndex setComponent(entity, BoneComponent, bone) addObjectToGroup(entity, bone) proxifyParentChildRelationships(bone) @@ -997,6 +1011,8 @@ const PrimitiveReactor = (props: { setComponent(entity, MeshComponent, mesh) } + mesh.name = node.name ?? 'Node-' + props.nodeIndex + /** @todo multiple primitive support */ addObjectToGroup(entity, mesh) proxifyParentChildRelationships(mesh) diff --git a/packages/engine/src/scene/components/ModelComponent.tsx b/packages/engine/src/scene/components/ModelComponent.tsx index b50ecf6565..1dc9d7104c 100644 --- a/packages/engine/src/scene/components/ModelComponent.tsx +++ b/packages/engine/src/scene/components/ModelComponent.tsx @@ -23,35 +23,31 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { QueryReactor, UUIDComponent } from '@etherealengine/ecs' +import { UUIDComponent } from '@etherealengine/ecs' import { defineComponent, getComponent, getOptionalComponent, hasComponent, setComponent, - useComponent, - useOptionalComponent + useComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { Engine } from '@etherealengine/ecs/src/Engine' -import { Entity, EntityUUID } from '@etherealengine/ecs/src/Entity' +import { EntityUUID } from '@etherealengine/ecs/src/Entity' import { useEntityContext } from '@etherealengine/ecs/src/EntityFunctions' import { NO_PROXY, dispatchAction, getMutableState, getState, none, useHookstate } from '@etherealengine/hyperflux' import { CameraComponent } from '@etherealengine/spatial/src/camera/components/CameraComponent' import { RendererComponent } from '@etherealengine/spatial/src/renderer/WebGLRendererSystem' import { GroupComponent, addObjectToGroup } from '@etherealengine/spatial/src/renderer/components/GroupComponent' -import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { ObjectLayerMaskComponent } from '@etherealengine/spatial/src/renderer/components/ObjectLayerComponent' import { ObjectLayers } from '@etherealengine/spatial/src/renderer/constants/ObjectLayers' import { EntityTreeComponent, iterateEntityNode, - removeEntityNodeRecursively, - useAncestorWithComponent + removeEntityNodeRecursively } from '@etherealengine/spatial/src/transform/components/EntityTree' import { VRM } from '@pixiv/three-vrm' -import { Not } from 'bitecs' -import React, { FC, useEffect } from 'react' +import { useEffect } from 'react' import { AnimationMixer, Group, Scene } from 'three' import { useGLTF } from '../../assets/functions/resourceLoaderHooks' import { GLTF } from '../../assets/loaders/gltf/GLTFLoader' @@ -62,7 +58,7 @@ import { GLTFSnapshotState, GLTFSourceState } from '../../gltf/GLTFState' import { SceneJsonType, convertSceneJSONToGLTF } from '../../gltf/convertJsonToGLTF' import { addError, removeError } from '../functions/ErrorFunctions' import { parseGLTFModel, proxifyParentChildRelationships } from '../functions/loadGLTFModel' -import { getModelSceneID, useModelSceneID } from '../functions/loaders/ModelFunctions' +import { getModelSceneID } from '../functions/loaders/ModelFunctions' import { SourceComponent } from './SourceComponent' /** @@ -245,43 +241,3 @@ function ModelReactor() { return null } - -/** - * Returns true if the entity has a model component or a mesh component that is not a child of model - * @param entity - * @returns {boolean} - */ -export const useHasModelOrIndependentMesh = (entity: Entity) => { - const hasModel = !!useOptionalComponent(entity, ModelComponent) - const isChildOfModel = !!useAncestorWithComponent(entity, ModelComponent) - const hasMesh = !!useOptionalComponent(entity, MeshComponent) - - return hasModel || (hasMesh && !isChildOfModel) -} - -export const MeshOrModelQuery = (props: { ChildReactor: FC<{ entity: Entity; rootEntity: Entity }> }) => { - const ModelReactor = () => { - const entity = useEntityContext() - const sceneInstanceID = useModelSceneID(entity) - const childEntities = useHookstate(SourceComponent.entitiesBySourceState[sceneInstanceID]) - return ( - <> - {childEntities.value?.map((childEntity) => ( - - ))} - - ) - } - - const MeshReactor = () => { - const entity = useEntityContext() - return - } - - return ( - <> - - - - ) -} diff --git a/packages/engine/src/scene/components/ObjectGridSnapComponent.ts b/packages/engine/src/scene/components/ObjectGridSnapComponent.ts index b4a1d15acb..b279edcd84 100644 --- a/packages/engine/src/scene/components/ObjectGridSnapComponent.ts +++ b/packages/engine/src/scene/components/ObjectGridSnapComponent.ts @@ -47,7 +47,7 @@ import { TransformComponent } from '@etherealengine/spatial/src/transform/compon import { computeTransformMatrix } from '@etherealengine/spatial/src/transform/systems/TransformSystem' import { useEffect } from 'react' import { Box3, BufferGeometry, ColorRepresentation, LineBasicMaterial, Matrix4, Mesh, Quaternion, Vector3 } from 'three' -import { ModelComponent } from './ModelComponent' +import { GLTFComponent } from '../../gltf/GLTFComponent' function createBBoxGridGeometry(matrixWorld: Matrix4, bbox: Box3, density: number): BufferGeometry { const lineSegmentList: Vector3[] = [] @@ -183,11 +183,12 @@ export const ObjectGridSnapComponent = defineComponent({ reactor: () => { const entity = useEntityContext() const engineState = useState(getMutableState(EngineState)) - const modelComponent = useComponent(entity, ModelComponent) + const modelComponent = useComponent(entity, GLTFComponent) + const modelLoaded = modelComponent.progress.value === 100 const snapComponent = useComponent(entity, ObjectGridSnapComponent) useEffect(() => { - if (!modelComponent.scene.value) return + if (!modelLoaded) return const originalPosition = new Vector3() const originalRotation = new Quaternion() const originalScale = new Vector3() @@ -227,7 +228,7 @@ export const ObjectGridSnapComponent = defineComponent({ iterateEntityNode(entity, computeTransformMatrix, (childEntity) => hasComponent(childEntity, TransformComponent)) //set bounding box in component snapComponent.bbox.set(bbox) - }, [modelComponent.scene]) + }, [modelLoaded]) useEffect(() => { if (!engineState.isEditing.value) return diff --git a/packages/engine/src/scene/functions/loaders/ModelFunctions.ts b/packages/engine/src/scene/functions/loaders/ModelFunctions.ts index ee1227323b..ebd4398143 100644 --- a/packages/engine/src/scene/functions/loaders/ModelFunctions.ts +++ b/packages/engine/src/scene/functions/loaders/ModelFunctions.ts @@ -27,12 +27,7 @@ import { DracoOptions } from '@gltf-transform/functions' import { Material, Texture } from 'three' import { UUIDComponent } from '@etherealengine/ecs' -import { - getComponent, - getOptionalComponent, - hasComponent, - useOptionalComponent -} from '@etherealengine/ecs/src/ComponentFunctions' +import { getComponent, getOptionalComponent, hasComponent } from '@etherealengine/ecs/src/ComponentFunctions' import { Entity } from '@etherealengine/ecs/src/Entity' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' import { iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' @@ -53,13 +48,6 @@ export function getModelSceneID(entity: Entity): string { return getComponent(entity, UUIDComponent) + '-' + getComponent(entity, ModelComponent).src } -export function useModelSceneID(entity: Entity): string { - const uuid = useOptionalComponent(entity, UUIDComponent)?.value - const model = useOptionalComponent(entity, ModelComponent)?.value - if (!uuid || !model) return '' - return uuid + '-' + model.src -} - export function getModelResources(entity: Entity, defaultParms: ModelTransformParameters): ResourceTransforms { const model = getOptionalComponent(entity, ModelComponent) if (!model?.scene) return { geometries: [], images: [] } diff --git a/packages/engine/src/scene/functions/migrateOldColliders.ts b/packages/engine/src/scene/functions/migrateOldColliders.ts index 6be7a9d019..258b09554b 100644 --- a/packages/engine/src/scene/functions/migrateOldColliders.ts +++ b/packages/engine/src/scene/functions/migrateOldColliders.ts @@ -27,17 +27,17 @@ import { ColliderComponent } from '@etherealengine/spatial/src/physics/component import { RigidBodyComponent } from '@etherealengine/spatial/src/physics/components/RigidBodyComponent' import { TriggerComponent } from '@etherealengine/spatial/src/physics/components/TriggerComponent' -import { ModelComponent } from '../components/ModelComponent' import { ComponentJsonType, EntityJsonType } from '../types/SceneTypes' const oldColliderJSONID = 'collider' +const oldModelJSONID = 'EE_model' /** * Converts old ColliderComponent to RigidbodyComponent, new ColliderComponent and TriggerComponent */ export const migrateOldColliders = (oldJSON: EntityJsonType) => { /** models need to be manually converted in the studio */ - const hasModel = Object.values(oldJSON.components).some((comp) => comp.name === ModelComponent.jsonID) + const hasModel = Object.values(oldJSON.components).some((comp) => comp.name === oldModelJSONID) if (hasModel) return const newComponents = [] as ComponentJsonType[] diff --git a/packages/engine/src/scene/materials/functions/materialSourcingFunctions.ts b/packages/engine/src/scene/materials/functions/materialSourcingFunctions.ts index 7eee24fcbb..73f6f99248 100644 --- a/packages/engine/src/scene/materials/functions/materialSourcingFunctions.ts +++ b/packages/engine/src/scene/materials/functions/materialSourcingFunctions.ts @@ -26,12 +26,12 @@ Ethereal Engine. All Rights Reserved. import { Entity, EntityUUID, getComponent, hasComponent } from '@etherealengine/ecs' import { MaterialInstanceComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' +import { GLTFComponent } from '../../../gltf/GLTFComponent' import { SourceComponent } from '../../components/SourceComponent' -import { getModelSceneID } from '../../functions/loaders/ModelFunctions' /**Gets all materials used by child and self entity */ export const getMaterialsFromScene = (source: Entity) => { - const sceneInstanceID = getModelSceneID(source) + const sceneInstanceID = GLTFComponent.getInstanceID(source) const childEntities = SourceComponent.entitiesBySource[sceneInstanceID] ?? ([] as Entity[]) childEntities.push(source) const materials = {} as Record diff --git a/packages/engine/src/scene/systems/SceneObjectSystem.tsx b/packages/engine/src/scene/systems/SceneObjectSystem.tsx index 25d82268b0..3b88e3d7b9 100644 --- a/packages/engine/src/scene/systems/SceneObjectSystem.tsx +++ b/packages/engine/src/scene/systems/SceneObjectSystem.tsx @@ -73,12 +73,11 @@ import { MaterialStateComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { createAndAssignMaterial } from '@etherealengine/spatial/src/renderer/materials/materialFunctions' +import { GLTFComponent } from '../../gltf/GLTFComponent' import { EnvmapComponent } from '../components/EnvmapComponent' -import { ModelComponent } from '../components/ModelComponent' import { ShadowComponent } from '../components/ShadowComponent' import { SourceComponent } from '../components/SourceComponent' import { UpdatableCallback, UpdatableComponent } from '../components/UpdatableComponent' -import { getModelSceneID, useModelSceneID } from '../functions/loaders/ModelFunctions' const disposeMaterial = (material: Material) => { for (const [key, val] of Object.entries(material) as [string, Texture][]) { @@ -168,8 +167,8 @@ function SceneObjectReactor(props: { entity: Entity; obj: Object3D }) { const forceBasicMaterials = useHookstate(renderState.forceBasicMaterials) useEffect(() => { - const source = hasComponent(entity, ModelComponent) - ? getModelSceneID(entity) + const source = hasComponent(entity, GLTFComponent) + ? GLTFComponent.getInstanceID(entity) : getOptionalComponent(entity, SourceComponent) return () => { ResourceManager.unloadObj(obj, source) @@ -210,8 +209,8 @@ const execute = () => { const ModelEntityReactor = () => { const entity = useEntityContext() - const modelSceneID = useModelSceneID(entity) - const childEntities = useHookstate(SourceComponent.entitiesBySourceState[modelSceneID]) + const modelInstanceID = GLTFComponent.useInstanceID(entity) + const childEntities = useHookstate(SourceComponent.entitiesBySourceState[modelInstanceID]) return ( <> @@ -275,7 +274,7 @@ const ChildReactor = (props: { entity: Entity; parentEntity: Entity }) => { const reactor = () => { return ( <> - + ) diff --git a/packages/engine/src/scene/systems/ShadowSystem.tsx b/packages/engine/src/scene/systems/ShadowSystem.tsx index 63a498a5c5..175cbf0751 100644 --- a/packages/engine/src/scene/systems/ShadowSystem.tsx +++ b/packages/engine/src/scene/systems/ShadowSystem.tsx @@ -91,8 +91,8 @@ import { EngineState } from '@etherealengine/spatial/src/EngineState' import { RenderModes } from '@etherealengine/spatial/src/renderer/constants/RenderModes' import { createDisposable } from '@etherealengine/spatial/src/resources/resourceHooks' import { useTexture } from '../../assets/functions/resourceLoaderHooks' +import { useHasModelOrIndependentMesh } from '../../gltf/GLTFComponent' import { DropShadowComponent } from '../components/DropShadowComponent' -import { useHasModelOrIndependentMesh } from '../components/ModelComponent' import { RenderSettingsComponent } from '../components/RenderSettingsComponent' import { ShadowComponent } from '../components/ShadowComponent' import { SceneObjectSystem } from './SceneObjectSystem' diff --git a/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts b/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts index 61888472e5..5e704b34ba 100644 --- a/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts +++ b/packages/spatial/src/renderer/components/SkinnedMeshComponent.ts @@ -52,7 +52,7 @@ import { VisibleComponent, setVisibleComponent } from '@etherealengine/spatial/s import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' import { EntityTreeComponent } from '@etherealengine/spatial/src/transform/components/EntityTree' import { useEffect } from 'react' -import { TransformComponent } from '../RendererModule' +import { TransformComponent } from '../../transform/components/TransformComponent' import { ObjectLayers } from '../constants/ObjectLayers' import { BoneComponent } from './BoneComponent' import { setObjectLayers } from './ObjectLayerComponent' diff --git a/packages/spatial/src/threejsPatches.ts b/packages/spatial/src/threejsPatches.ts index 604b6907e3..29eca7835c 100644 --- a/packages/spatial/src/threejsPatches.ts +++ b/packages/spatial/src/threejsPatches.ts @@ -24,13 +24,29 @@ Ethereal Engine. All Rights Reserved. */ import * as THREE from 'three' -import { Euler, Matrix4, Object3D, Quaternion, Scene, SkinnedMesh, Vector2, Vector3, Vector4 } from 'three' +import { + Euler, + Matrix4, + Object3D, + PropertyBinding, + Quaternion, + Scene, + SkinnedMesh, + Vector2, + Vector3, + Vector4 +} from 'three' import { isClient } from '@etherealengine/common/src/utils/getEnvironment' import { Object3DUtils } from '@etherealengine/common/src/utils/Object3DUtils' -import { Entity } from '@etherealengine/ecs' +import { Entity, getComponent, getOptionalComponent, hasComponent } from '@etherealengine/ecs' import { overrideOnBeforeCompile } from './common/functions/OnBeforeCompilePlugin' +import { BoneComponent } from './renderer/components/BoneComponent' +import { MeshComponent } from './renderer/components/MeshComponent' +import { Object3DComponent } from './renderer/components/Object3DComponent' +import { SkinnedMeshComponent } from './renderer/components/SkinnedMeshComponent' +import { EntityTreeComponent } from './transform/components/EntityTree' //@ts-ignore Vector3.prototype.toJSON = function () { @@ -214,6 +230,63 @@ SkinnedMesh.prototype.applyBoneTransform = function (index, vector) { return vector.applyMatrix4(this.bindMatrixInverse) } +PropertyBinding.findNode = function (root: SkinnedMesh, nodeName: string | number) { + if ( + nodeName === undefined || + nodeName === '' || + nodeName === '.' || + nodeName === -1 || + nodeName === root.name || + nodeName === root.uuid + ) { + return root + } + + // search into skeleton bones. + if (root.skeleton) { + const bone = root.skeleton.getBoneByName(nodeName as string) + + if (bone !== undefined) { + return bone + } + } + + const entity = root.entity + if (!hasComponent(entity, EntityTreeComponent)) return null + + const children = getComponent(entity, EntityTreeComponent).children + + // search into node subtree. + const searchNodeSubtree = function (children: Entity[]) { + for (let i = 0; i < children.length; i++) { + const entity = children[i] + const childNode = + getOptionalComponent(entity, BoneComponent) ?? + getOptionalComponent(entity, MeshComponent) ?? + getOptionalComponent(entity, SkinnedMeshComponent) ?? + getOptionalComponent(entity, Object3DComponent)! + + if (childNode && (childNode.name === nodeName || childNode.uuid === nodeName)) { + return childNode + } + + const result = searchNodeSubtree(getComponent(entity, EntityTreeComponent).children) + + if (result) return result + } + + return null + } + + const subTreeNode = searchNodeSubtree(children) + + if (subTreeNode) { + return subTreeNode + } + + return null +} + overrideOnBeforeCompile() globalThis.THREE = { ...THREE } as any diff --git a/packages/spatial/src/transform/components/EntityTree.tsx b/packages/spatial/src/transform/components/EntityTree.tsx index 41feecb691..43dafcfa8b 100644 --- a/packages/spatial/src/transform/components/EntityTree.tsx +++ b/packages/spatial/src/transform/components/EntityTree.tsx @@ -291,6 +291,34 @@ export function getAncestorWithComponent( return result } +const inlineMatchesQuery = (entity: Entity, components: ComponentType[]) => + components.map((c) => hasComponent(entity, c)).filter((c) => !!c).length === components.length + +/** + * Returns the closest ancestor of an entity that has the given component by walking up the entity tree + * @param entity Entity to start from + * @param component Component to search for + * @param closest (default true) - whether to return the closest ancestor or the furthest ancestor + * @param includeSelf (default true) - whether to include the entity itself in the search + * @returns + */ +export function getAncestorWithComponents( + entity: Entity, + components: ComponentType, + closest = true, + includeSelf = true +): Entity { + let result = UndefinedEntity + if (includeSelf && closest && inlineMatchesQuery(entity, components)) return entity + traverseEntityNodeParent(entity, (parent) => { + if (closest && result) return + if (inlineMatchesQuery(parent, components)) { + result = parent + } + }) + return result +} + /** * Finds the index of an entity tree node using entity. * This function is useful for node which is not contained in array but can have same entity as one of array elements @@ -376,6 +404,7 @@ export function useTreeQuery(entity: Entity) { /** * Returns the closest ancestor of an entity that has a component + * @deprecated use useAncestorWithComponents instead * @todo maybe extend this to be a list of components? * @todo maybe extend this or write an alternative to get the furthest ancestor with component? * @param entity @@ -419,6 +448,52 @@ export function useAncestorWithComponent(entity: Entity, component: ComponentTyp return result.value } +/** + * Returns the closest ancestor of an entity that has a component + * @todo maybe extend this to be a list of components? + * @todo maybe extend this or write an alternative to get the furthest ancestor with component? + * @param entity + * @param component + * @param closest + * @returns + */ +export function useAncestorWithComponents(entity: Entity, components: ComponentType[]) { + const result = useHookstate(() => getAncestorWithComponents(entity, components)) + + useImmediateEffect(() => { + let unmounted = false + const ParentSubReactor = (props: { entity: Entity }) => { + const tree = useOptionalComponent(props.entity, EntityTreeComponent) + const matchesQuery = + components.map((c) => useOptionalComponent(props.entity, c)).filter((c) => !!c).length === components.length + + useLayoutEffect(() => { + if (!matchesQuery) return + result.set(props.entity) + return () => { + if (!unmounted) result.set(UndefinedEntity) + } + }, [tree?.parentEntity?.value, matchesQuery]) + + if (matchesQuery) return null + + if (!tree?.parentEntity?.value) return null + + return + } + + const root = startReactor(function useQueryReactor() { + return + }) + return () => { + unmounted = true + root.stop() + } + }, [entity, components.map((c) => c.name).join(',')]) + + return result.value +} + /** * @todo - return an array of entities that have the component * From 8b6ed070913c91dd73670fa6a78a7fc826e0f6b5 Mon Sep 17 00:00:00 2001 From: HexaField Date: Thu, 15 Aug 2024 19:42:40 +1000 Subject: [PATCH 44/47] license --- .../src/avatar/components/VRMComponent.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/engine/src/avatar/components/VRMComponent.ts b/packages/engine/src/avatar/components/VRMComponent.ts index c1424c5edd..e761f9d84c 100644 --- a/packages/engine/src/avatar/components/VRMComponent.ts +++ b/packages/engine/src/avatar/components/VRMComponent.ts @@ -1,3 +1,28 @@ +/* +CPAL-1.0 License + +The contents of this file are subject to the Common Public Attribution License +Version 1.0. (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at +https://github.com/EtherealEngine/etherealengine/blob/dev/LICENSE. +The License is based on the Mozilla Public License Version 1.1, but Sections 14 +and 15 have been added to cover use of software over a computer network and +provide for limited attribution for the Original Developer. In addition, +Exhibit A has been modified to be consistent with Exhibit B. + +Software distributed under the License is distributed on an "AS IS" basis, +WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the +specific language governing rights and limitations under the License. + +The Original Code is Ethereal Engine. + +The Original Developer is the Initial Developer. The Initial Developer of the +Original Code is the Ethereal Engine team. + +All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023 +Ethereal Engine. All Rights Reserved. +*/ + import { defineComponent } from '@etherealengine/ecs' import { VRM } from '@pixiv/three-vrm' From 5a674d669d4ce86f32897ca4aaf7311abae781e1 Mon Sep 17 00:00:00 2001 From: HexaField Date: Fri, 16 Aug 2024 13:26:31 +1000 Subject: [PATCH 45/47] fix regression with avatar animation track binding --- .../engine/src/gltf/GLTFLoaderFunctions.ts | 7 +-- packages/spatial/src/threejsPatches.ts | 53 ++++++++++++++----- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/packages/engine/src/gltf/GLTFLoaderFunctions.ts b/packages/engine/src/gltf/GLTFLoaderFunctions.ts index 0aee5e3ad2..747ca9fdba 100644 --- a/packages/engine/src/gltf/GLTFLoaderFunctions.ts +++ b/packages/engine/src/gltf/GLTFLoaderFunctions.ts @@ -35,7 +35,6 @@ import { NO_PROXY, getState, startReactor, useHookstate } from '@etherealengine/ import { NameComponent } from '@etherealengine/spatial/src/common/NameComponent' import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { MeshComponent } from '@etherealengine/spatial/src/renderer/components/MeshComponent' -import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' import { MaterialPrototypeComponent } from '@etherealengine/spatial/src/renderer/materials/MaterialComponent' import { GLTF } from '@gltf-transform/core' import { useEffect } from 'react' @@ -986,17 +985,15 @@ const useLoadAnimation = (options: GLTFParserOptions, animationIndex?: number) = /** @todo we should probably jsut use GroupComponent or something here once we stop creating Object3Ds for all nodes */ const meshComponent = useOptionalComponent(targetNodeEntity, MeshComponent) const boneComponent = useOptionalComponent(targetNodeEntity, BoneComponent) - const obj3dComponent = useOptionalComponent(targetNodeEntity, Object3DComponent) useEffect(() => { const meshWeightsLoaded = meshHasWeights ? meshComponent?.get(NO_PROXY)?.morphTargetInfluences !== undefined : true - if (!meshWeightsLoaded && !boneComponent && !obj3dComponent) return + if (!meshWeightsLoaded && !boneComponent) return channelData[i].nodes.set( getOptionalComponent(targetNodeEntity, MeshComponent) ?? - getOptionalComponent(targetNodeEntity, BoneComponent) ?? - getOptionalComponent(targetNodeEntity, Object3DComponent)! + getOptionalComponent(targetNodeEntity, BoneComponent)! ) }, [meshComponent, boneComponent]) diff --git a/packages/spatial/src/threejsPatches.ts b/packages/spatial/src/threejsPatches.ts index 29eca7835c..1fe4b217a7 100644 --- a/packages/spatial/src/threejsPatches.ts +++ b/packages/spatial/src/threejsPatches.ts @@ -252,25 +252,50 @@ PropertyBinding.findNode = function (root: SkinnedMesh, nodeName: string | numbe } const entity = root.entity - if (!hasComponent(entity, EntityTreeComponent)) return null + if (entity) { + if (!hasComponent(entity, EntityTreeComponent)) return null + + const children = getComponent(entity, EntityTreeComponent).children + + // search into node subtree. + const searchEntitySubtree = function (children: Entity[]) { + for (let i = 0; i < children.length; i++) { + const entity = children[i] + const childNode = + getOptionalComponent(entity, BoneComponent) ?? + getOptionalComponent(entity, MeshComponent) ?? + getOptionalComponent(entity, SkinnedMeshComponent) ?? + getOptionalComponent(entity, Object3DComponent)! + + if (childNode && (childNode.name === nodeName || childNode.uuid === nodeName)) { + return childNode + } + + const result = searchEntitySubtree(getComponent(entity, EntityTreeComponent).children) + + if (result) return result + } + + return null + } - const children = getComponent(entity, EntityTreeComponent).children + const subTreeNode = searchEntitySubtree(children) - // search into node subtree. - const searchNodeSubtree = function (children: Entity[]) { + if (subTreeNode) { + return subTreeNode + } + } + + // fallback to three hierarchy for non-ecs hierarchy (normalize vrm rigs) + const searchNodeSubtree = function (children) { for (let i = 0; i < children.length; i++) { - const entity = children[i] - const childNode = - getOptionalComponent(entity, BoneComponent) ?? - getOptionalComponent(entity, MeshComponent) ?? - getOptionalComponent(entity, SkinnedMeshComponent) ?? - getOptionalComponent(entity, Object3DComponent)! - - if (childNode && (childNode.name === nodeName || childNode.uuid === nodeName)) { + const childNode = children[i] + + if (childNode.name === nodeName || childNode.uuid === nodeName) { return childNode } - const result = searchNodeSubtree(getComponent(entity, EntityTreeComponent).children) + const result = searchNodeSubtree(childNode.children) if (result) return result } @@ -278,7 +303,7 @@ PropertyBinding.findNode = function (root: SkinnedMesh, nodeName: string | numbe return null } - const subTreeNode = searchNodeSubtree(children) + const subTreeNode = searchNodeSubtree(root.children) if (subTreeNode) { return subTreeNode From a80ec6047ea13c084e6522640350ec27d0d3b50b Mon Sep 17 00:00:00 2001 From: AidanCaruso Date: Fri, 16 Aug 2024 15:04:48 -0400 Subject: [PATCH 46/47] ready player me almost works --- .../components/AvatarAnimationComponent.ts | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index a2960a2cfa..cb2391cdca 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -40,6 +40,7 @@ import { AnimationAction, Euler, Group, Matrix4, Vector3 } from 'three' import { defineComponent, getComponent, + getOptionalComponent, hasComponent, removeComponent, setComponent, @@ -54,6 +55,7 @@ import { addObjectToGroup } from '@etherealengine/spatial/src/renderer/component import { ComputedTransformComponent } from '@etherealengine/spatial/src/transform/components/ComputedTransformComponent' import { UUIDComponent } from '@etherealengine/ecs' +import { TransformComponent } from '@etherealengine/spatial' import { BoneComponent } from '@etherealengine/spatial/src/renderer/components/BoneComponent' import { Object3DComponent } from '@etherealengine/spatial/src/renderer/components/Object3DComponent' import { EntityTreeComponent, iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree' @@ -201,6 +203,8 @@ export const AvatarRigComponent = defineComponent({ const _rightHandPos = new Vector3(), _rightUpperArmPos = new Vector3() +const flip = new Matrix4().makeRotationFromEuler(new Euler(0, Math.PI, 0)) + export default function createVRM(rootEntity: Entity) { const documentID = GLTFComponent.getInstanceID(rootEntity) const gltf = getState(GLTFDocumentState)[documentID] @@ -211,10 +215,10 @@ export default function createVRM(rootEntity: Entity) { addObjectToGroup(rootEntity, obj3d) proxifyParentChildRelationships(obj3d) } - + console.log(gltf) if (gltf.extensions?.VRM) { const vrmExtensionDefinition = gltf.extensions!.VRM as V0VRM.VRM - const flip = new Matrix4().makeRotationFromEuler(new Euler(0, Math.PI, 0)) + console.log(vrmExtensionDefinition) const bones = vrmExtensionDefinition.humanoid!.humanBones!.reduce((bones, bone) => { const nodeID = `${documentID}-${bone.node}` as EntityUUID const entity = UUIDComponent.getEntityByUUID(nodeID) @@ -224,13 +228,14 @@ export default function createVRM(rootEntity: Entity) { /**hacky, @todo test with vrm1 */ iterateEntityNode(bones.hips.node.parent!.entity, (entity) => { - const bone = getComponent(entity, BoneComponent) - bone.matrixWorld.identity() - if (bone.entity != bones.hips.node.parent!.entity) bone.matrixWorld.multiply(flip) + const bone = getOptionalComponent(entity, BoneComponent) + bone?.matrixWorld.identity() + if (bone && bone?.entity != bones.hips.node.parent!.entity && vrmExtensionDefinition.meta?.version === '0') + bone.matrixWorld.multiply(flip) }) bones.hips.node.rotateY(Math.PI) - const humanoid = new VRMHumanoid(bones) + const humanoid = enforceTPose(bones) const scene = getComponent(rootEntity, Object3DComponent) as any as Group @@ -288,8 +293,13 @@ const createVRMFromGLTF = (rootEntity: Entity, gltf: GLTF.IGLTF) => { const removeSuffix = mixamoPrefix ? false : !/[hp]/i.test(hipsName.charAt(9)) console.log({ removeSuffix, mixamoPrefix }) - iterateEntityNode(hipsEntity, (entity) => { + iterateEntityNode(getComponent(hipsEntity, EntityTreeComponent).parentEntity, (entity) => { // if (!getComponent(entity, BoneComponent)) return + const boneComponent = getOptionalComponent(entity, BoneComponent) || getComponent(entity, TransformComponent) + boneComponent?.matrixWorld.identity() + console.log(boneComponent) + if (boneComponent && entity != getComponent(hipsEntity, EntityTreeComponent).parentEntity && entity != hipsEntity) + boneComponent.matrixWorld.multiply(flip) const name = getComponent(entity, NameComponent) /**match the keys to create a humanoid bones object */ @@ -302,8 +312,7 @@ const createVRMFromGLTF = (rootEntity: Entity, gltf: GLTF.IGLTF) => { } }) - console.log(bones) - const humanoid = enforceTPose(new VRMHumanoid(bones)) + const humanoid = enforceTPose(bones) const scene = getComponent(rootEntity, Object3DComponent) const children = getComponent(rootEntity, EntityTreeComponent).children const childName = getComponent(children[0], NameComponent) @@ -329,9 +338,8 @@ const createVRMFromGLTF = (rootEntity: Entity, gltf: GLTF.IGLTF) => { const legAngle = new Euler(0, 0, Math.PI) const rightShoulderAngle = new Euler(Math.PI / 2, 0, Math.PI / 2) const leftShoulderAngle = new Euler(Math.PI / 2, 0, -Math.PI / 2) -export const enforceTPose = (humanoid: VRMHumanoid) => { - const bones = humanoid.humanBones - console.log('enforcing T pose', humanoid, bones) +export const enforceTPose = (bones: VRMHumanBones) => { + console.log('enforcing T pose', bones) bones.rightShoulder!.node.quaternion.setFromEuler(rightShoulderAngle) bones.rightUpperArm.node.quaternion.set(0, 0, 0, 1) From 3f949541a3f8d117f6c731f60d3e3b69b6416f02 Mon Sep 17 00:00:00 2001 From: AidanCaruso Date: Sat, 17 Aug 2024 17:27:23 -0400 Subject: [PATCH 47/47] fix world matrix calculations in tpose enforcement function --- .../components/AvatarAnimationComponent.ts | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts index cb2391cdca..1bae2d36d1 100755 --- a/packages/engine/src/avatar/components/AvatarAnimationComponent.ts +++ b/packages/engine/src/avatar/components/AvatarAnimationComponent.ts @@ -230,12 +230,11 @@ export default function createVRM(rootEntity: Entity) { iterateEntityNode(bones.hips.node.parent!.entity, (entity) => { const bone = getOptionalComponent(entity, BoneComponent) bone?.matrixWorld.identity() - if (bone && bone?.entity != bones.hips.node.parent!.entity && vrmExtensionDefinition.meta?.version === '0') - bone.matrixWorld.multiply(flip) + bone?.matrixWorld.makeRotationY(Math.PI) }) - bones.hips.node.rotateY(Math.PI) + bones.hips.node.parent!.rotateY(Math.PI) - const humanoid = enforceTPose(bones) + const humanoid = new VRMHumanoid(bones) const scene = getComponent(rootEntity, Object3DComponent) as any as Group @@ -282,7 +281,7 @@ const createVRMFromGLTF = (rootEntity: Entity, gltf: GLTF.IGLTF) => { const bones = {} as VRMHumanBones /** - * some mixamo rigs do not use the mixamo prefix, if so we add + * some mixamo rigs do not use the mixamo prefix, if they don't, we add * a prefix to the rig names for matching to keys in the mixamoVRMRigMap */ const mixamoPrefix = hipsName.includes('mixamorig') ? '' : 'mixamorig' @@ -291,27 +290,30 @@ const createVRMFromGLTF = (rootEntity: Entity, gltf: GLTF.IGLTF) => { * that must be removed for matching to keys in the mixamoVRMRigMap */ const removeSuffix = mixamoPrefix ? false : !/[hp]/i.test(hipsName.charAt(9)) - console.log({ removeSuffix, mixamoPrefix }) - iterateEntityNode(getComponent(hipsEntity, EntityTreeComponent).parentEntity, (entity) => { + iterateEntityNode(hipsEntity, (entity) => { // if (!getComponent(entity, BoneComponent)) return const boneComponent = getOptionalComponent(entity, BoneComponent) || getComponent(entity, TransformComponent) boneComponent?.matrixWorld.identity() - console.log(boneComponent) - if (boneComponent && entity != getComponent(hipsEntity, EntityTreeComponent).parentEntity && entity != hipsEntity) - boneComponent.matrixWorld.multiply(flip) + if (entity != hipsEntity) { + boneComponent.matrixWorld.makeRotationZ(Math.PI) + } const name = getComponent(entity, NameComponent) /**match the keys to create a humanoid bones object */ let boneName = mixamoPrefix + name + if (removeSuffix) boneName = boneName.slice(0, 9) + name.slice(10) + + //remove colon from the bone name + if (boneName.includes(':')) boneName = boneName.replace(':', '') + const bone = mixamoVRMRigMap[boneName] as string console.log({ name, boneName, removeSuffix, bone, mixamoVRMRigMap }) if (bone) { bones[bone] = { node: getComponent(entity, BoneComponent) } as VRMHumanBone } }) - const humanoid = enforceTPose(bones) const scene = getComponent(rootEntity, Object3DComponent) const children = getComponent(rootEntity, EntityTreeComponent).children @@ -342,17 +344,30 @@ export const enforceTPose = (bones: VRMHumanBones) => { console.log('enforcing T pose', bones) bones.rightShoulder!.node.quaternion.setFromEuler(rightShoulderAngle) + iterateEntityNode(bones.rightShoulder!.node.entity, (entity) => { + getComponent(entity, BoneComponent).matrixWorld.makeRotationFromEuler(rightShoulderAngle) + }) + bones.rightShoulder!.node.matrixWorld.makeRotationFromEuler(rightShoulderAngle) bones.rightUpperArm.node.quaternion.set(0, 0, 0, 1) bones.rightLowerArm.node.quaternion.set(0, 0, 0, 1) bones.leftShoulder!.node.quaternion.setFromEuler(leftShoulderAngle) + iterateEntityNode(bones.leftShoulder!.node.entity, (entity) => { + getComponent(entity, BoneComponent).matrixWorld.makeRotationFromEuler(leftShoulderAngle) + }) bones.leftUpperArm.node.quaternion.set(0, 0, 0, 1) bones.leftLowerArm.node.quaternion.set(0, 0, 0, 1) bones.rightUpperLeg.node.quaternion.setFromEuler(legAngle) + iterateEntityNode(bones.rightUpperLeg!.node.entity, (entity) => { + getComponent(entity, BoneComponent).matrixWorld.makeRotationFromEuler(legAngle) + }) bones.rightLowerLeg.node.quaternion.set(0, 0, 0, 1) bones.leftUpperLeg.node.quaternion.setFromEuler(legAngle) + iterateEntityNode(bones.leftUpperLeg!.node.entity, (entity) => { + getComponent(entity, BoneComponent).matrixWorld.makeRotationFromEuler(legAngle) + }) bones.leftLowerLeg.node.quaternion.set(0, 0, 0, 1) return new VRMHumanoid(bones)