From 376487876acde1bd0cb77c9b4b31959f0d872c66 Mon Sep 17 00:00:00 2001 From: Diego Marcos Segura Date: Tue, 14 Nov 2023 02:20:32 -0800 Subject: [PATCH] Add support for WebXR Mesh and WebXR Plane modules --- .eslintrc.json | 7 + docs/components/real-world-meshing.md | 29 +++ examples/index.html | 1 + examples/js/center-model.js | 30 +++ .../real-world-meshing/coffee-spawner.js | 93 ++++++++ .../real-world-meshing/hide-model-parts.js | 43 ++++ .../real-world-meshing/index.html | 50 +++++ .../real-world-meshing/message.html | 11 + src/components/index.js | 1 + src/components/scene/real-world-meshing.js | 212 ++++++++++++++++++ 10 files changed, 477 insertions(+) create mode 100644 docs/components/real-world-meshing.md create mode 100644 examples/js/center-model.js create mode 100644 examples/mixed-reality/real-world-meshing/coffee-spawner.js create mode 100644 examples/mixed-reality/real-world-meshing/hide-model-parts.js create mode 100644 examples/mixed-reality/real-world-meshing/index.html create mode 100644 examples/mixed-reality/real-world-meshing/message.html create mode 100644 src/components/scene/real-world-meshing.js diff --git a/.eslintrc.json b/.eslintrc.json index e888c1a0989..4bbff20a404 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -54,6 +54,13 @@ "ecmaVersion": 6 } }, + { + /* This module uses ES6 */ + "files": ["./src/components/scene/real-world-meshing.js"], + "parserOptions": { + "ecmaVersion": 6 + } + }, { /* This module uses ES8 async / await due to WebXR Anchor Module integration */ "files": ["./src/components/anchored.js"], diff --git a/docs/components/real-world-meshing.md b/docs/components/real-world-meshing.md new file mode 100644 index 00000000000..946f01e8d45 --- /dev/null +++ b/docs/components/real-world-meshing.md @@ -0,0 +1,29 @@ +--- +title: real-world-meshing +type: components +layout: docs +parent_section: components +source_code: src/components/scene/real-world-meshing.js +examples: [] +--- + +Set this component on the scene element to render meshes corresponding to 3D surfaces detected in user's enviornment: this includes planes and meshes corresponding to floor, ceiling, walls and other objects. Each plane or meshes comes with a label indicating the type of surface or object. + +This component requires a browser with support for the [WebXR Mesh Detection Module]([object +pooling](https://en.wikipedia.org/wiki/Object_pool_pattern) and the [WebXR Plane Detection Module](https://immersive-web.github.io/real-world-geometry/plane-detection.html). The system / headset used might require additional scene setup by the use like setting up floor, walls, ceiling or labeling furniture in the space. + +## Example + +```html + +``` + +## Properties + +| Property | Description | Default Value +|---------------|---------------------------------------------------------------------------------------|--------------- +| filterLabels | List of labels corresponding to the surfaces that will be rendered. Can constrain rendering to certain surfaces like desks, walls, tables... All surfaces will be rendered if left empty. | [] | +| meshesEnabled | If meshes will be rendered as returned by the WebXR Mesh Detection Module. | true | +| meshMixin | Mixin applied to the entities corresponding to the detected meshes. | '' | +| planesEnabled | If planes will be rendered as returned by the WebXR Plane Detection Module. | true | +| planeMixin | Mixin applied to the entities corresponding to the detected planes. | '' | \ No newline at end of file diff --git a/examples/index.html b/examples/index.html index 08246e6675c..4ac50dfbf21 100644 --- a/examples/index.html +++ b/examples/index.html @@ -155,6 +155,7 @@

Examples

  • 360° Video
  • 3D Model (glTF)
  • Anchor (Mixed Reality)
    • +
  • Real World Meshing (Mixed Reality)

    • Examples from Documentation

    diff --git a/examples/js/center-model.js b/examples/js/center-model.js new file mode 100644 index 00000000000..1e8ccf9b390 --- /dev/null +++ b/examples/js/center-model.js @@ -0,0 +1,30 @@ +/* global AFRAME, THREE */ +AFRAME.registerComponent('center-model', { + init: function () { + var model; + this.onModelLoaded = this.onModelLoaded.bind(this); + model = this.el.components['gltf-model'] && this.el.components['gltf-model'].model; + if (model) { + this.centerModel(model); + return; + } + document.addEventListener('model-loaded', this.onModelLoaded); + }, + + onModelLoaded: function () { + var model = this.el.components['gltf-model'].model; + this.centerModel(model); + }, + + centerModel: function (model) { + var box; + var center; + this.el.removeObject3D('mesh'); + box = new THREE.Box3().setFromObject(model); + center = box.getCenter(new THREE.Vector3()); + model.position.x += (model.position.x - center.x); + model.position.y += (model.position.y - center.y); + model.position.z += (model.position.z - center.z); + this.el.setObject3D('mesh', model); + } +}); diff --git a/examples/mixed-reality/real-world-meshing/coffee-spawner.js b/examples/mixed-reality/real-world-meshing/coffee-spawner.js new file mode 100644 index 00000000000..934e5f73377 --- /dev/null +++ b/examples/mixed-reality/real-world-meshing/coffee-spawner.js @@ -0,0 +1,93 @@ +/* global AFRAME */ +AFRAME.registerComponent('coffee-spawner', { + schema: { + delay: {default: 250}, + targetElementSelector: {default: ''} + }, + init: function () { + var el = this.el; + this.delaySpawn = this.delaySpawn.bind(this); + this.cancelDelayedSpawn = this.cancelDelayedSpawn.bind(this); + this.onCollisionEnded = this.onCollisionEnded.bind(this); + this.onCollisionStarted = this.onCollisionStarted.bind(this); + this.el.addEventListener('pinchstarted', this.delaySpawn); + this.el.addEventListener('pinchended', this.cancelDelayedSpawn); + el.addEventListener('obbcollisionstarted', this.onCollisionStarted); + el.addEventListener('obbcollisionended', this.onCollisionEnded); + this.enabled = true; + }, + + delaySpawn: function (evt) { + this.pinchEvt = evt; + this.spawnDelay = this.data.delay; + }, + + cancelDelayedSpawn: function (evt) { + this.spawnDelay = undefined; + }, + + tick: function (time, delta) { + if (!this.spawnDelay) { return; } + this.spawnDelay -= delta; + if (this.spawnDelay <= 0) { + this.spawn(this.pinchEvt); + this.spawnDelay = undefined; + } + }, + + spawn: function (evt) { + var auxEuler = this.auxEuler; + var sceneEl = this.el.sceneEl; + var saucerEl = document.createElement('a-entity'); + var cupEl = document.createElement('a-entity'); + var wristRotation = evt.detail.wristRotation; + var animateScale = function (evt) { + evt.target.setAttribute('animation', { + property: 'scale', + from: {x: 0, y: 0, z: 0}, + to: {x: 0.0015, y: 0.0015, z: 0.0015}, + dur: 200 + }); + }; + + if (!this.enabled) { return; } + if (this.data.targetElementSelector && !this.targetEl) { return; } + + saucerEl.setAttribute('gltf-model', '#coffee'); + saucerEl.setAttribute('grabbable', ''); + saucerEl.setAttribute('hide-model-parts', 'parts: coffee, cup, handle'); + saucerEl.setAttribute('scale', '0.0015 0.0015 0.0015'); + saucerEl.setAttribute('position', evt.detail.position); + saucerEl.addEventListener('loaded', animateScale); + sceneEl.appendChild(saucerEl); + + cupEl.setAttribute('gltf-model', '#coffee'); + cupEl.setAttribute('grabbable', ''); + cupEl.setAttribute('hide-model-parts', 'parts: saucer'); + cupEl.setAttribute('rotation', '0 90 0'); + cupEl.setAttribute('scale', '0.0015 0.0015 0.0015'); + cupEl.setAttribute('position', evt.detail.position); + cupEl.addEventListener('loaded', animateScale); + sceneEl.appendChild(cupEl); + }, + + onCollisionStarted: function (evt) { + var targetElementSelector = this.data.targetElementSelector; + var targetEl = targetElementSelector && this.el.sceneEl.querySelector(targetElementSelector); + if (targetEl === evt.detail.withEl) { + this.targetEl = targetEl; + return; + } + this.enabled = false; + }, + + onCollisionEnded: function (evt) { + var targetElementSelector = this.data.targetElementSelector; + var targetEl = targetElementSelector && this.el.sceneEl.querySelector(targetElementSelector); + if (targetEl === evt.detail.withEl) { + this.targetEl = undefined; + return; + } + this.enabled = true; + } +}); diff --git a/examples/mixed-reality/real-world-meshing/hide-model-parts.js b/examples/mixed-reality/real-world-meshing/hide-model-parts.js new file mode 100644 index 00000000000..eea1d3c172f --- /dev/null +++ b/examples/mixed-reality/real-world-meshing/hide-model-parts.js @@ -0,0 +1,43 @@ +/* global AFRAME, THREE */ +AFRAME.registerComponent('hide-model-parts', { + schema: { + parts: {type: 'array'} + }, + + update: function () { + this.hideParts = this.hideParts.bind(this); + this.el.addEventListener('model-loaded', this.hideParts); + }, + + hideParts: function () { + var part; + var parts = this.data.parts; + var model = this.el.getObject3D('mesh'); + for (var i = 0; i < parts.length; i++) { + part = model.getObjectByName(parts[i]); + part.parent.remove(part); + } + }, + + /** + * Search for the part name and look for a mesh. + */ + selectFromModel: function (model) { + var mesh; + var part; + part = model.getObjectByName(this.data.part); + if (!part) { + console.error('[gltf-part] `' + this.data.part + '` not found in model.'); + return; + } + + mesh = part.getObjectByProperty('type', 'Mesh').clone(true); + + if (this.data.buffer) { + mesh.geometry = mesh.geometry.toNonIndexed(); + return mesh; + } + mesh.geometry = new THREE.Geometry().fromBufferGeometry(mesh.geometry); + return mesh; + } +}); diff --git a/examples/mixed-reality/real-world-meshing/index.html b/examples/mixed-reality/real-world-meshing/index.html new file mode 100644 index 00000000000..8f5cb127d40 --- /dev/null +++ b/examples/mixed-reality/real-world-meshing/index.html @@ -0,0 +1,50 @@ + + + + + Real World Meshing (Mixed Reality) • A-Frame + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/mixed-reality/real-world-meshing/message.html b/examples/mixed-reality/real-world-meshing/message.html new file mode 100644 index 00000000000..72deeda4689 --- /dev/null +++ b/examples/mixed-reality/real-world-meshing/message.html @@ -0,0 +1,11 @@ +

    +This demo requires a browser supporting the WebXR Mesh Detection Module and WebXR Plane Detection Module +

    + +

    +In AR mode the user can pinch over a suface labeled as table to spawn coffee cups. This requires the user to setup the environment: set boundaries, mesh and label at least one surface as table. +

    + +

    +Frame model by Rosnandie Yikie +

    \ No newline at end of file diff --git a/src/components/index.js b/src/components/index.js index 492a67cf6e4..15cbb092751 100644 --- a/src/components/index.js +++ b/src/components/index.js @@ -50,6 +50,7 @@ require('./scene/inspector'); require('./scene/fog'); require('./scene/keyboard-shortcuts'); require('./scene/pool'); +require('./scene/real-world-meshing'); require('./scene/reflection'); require('./scene/screenshot'); require('./scene/stats'); diff --git a/src/components/scene/real-world-meshing.js b/src/components/scene/real-world-meshing.js new file mode 100644 index 00000000000..76fdb1e1932 --- /dev/null +++ b/src/components/scene/real-world-meshing.js @@ -0,0 +1,212 @@ +/* global XRPlane, XRMesh */ +var register = require('../../core/component').registerComponent; +var THREE = require('../../lib/three'); + +/** + * Real World Meshing. + * + * Create entities with meshes corresponding to 3D surfaces detected in user's enviornment. + * It requires a browser with support for the WebXR Mesh and Plane detection modules. + * + */ +module.exports.Component = register('real-world-meshing', { + schema: { + filterLabels: {type: 'array'}, + meshesEnabled: {default: true}, + meshMixin: {default: true}, + planesEnabled: {default: true}, + planeMixin: {default: ''} + }, + + init: function () { + var webxrData = this.el.getAttribute('webxr'); + var requiredFeaturesArray = webxrData.requiredFeatures; + if (requiredFeaturesArray.indexOf('mesh-detection') === -1) { + requiredFeaturesArray.push('mesh-detection'); + this.el.setAttribute('webxr', webxrData); + } + if (requiredFeaturesArray.indexOf('plane-detection') === -1) { + requiredFeaturesArray.push('plane-detection'); + this.el.setAttribute('webxr', webxrData); + } + this.meshEntities = []; + this.initWorldMeshEntity = this.initWorldMeshEntity.bind(this); + }, + + tick: function () { + if (!this.el.is('ar-mode')) { return; } + this.detectMeshes(); + this.updateMeshes(); + }, + + detectMeshes: function () { + var data = this.data; + var detectedMeshes; + var detectedPlanes; + var sceneEl = this.el; + var xrManager = sceneEl.renderer.xr; + var frame; + var meshEntities = this.meshEntities; + var present = false; + var newMeshes = []; + var filterLabels = this.data.filterLabels; + + frame = sceneEl.frame; + detectedMeshes = frame.detectedMeshes; + detectedPlanes = frame.detectedPlanes; + + for (var i = 0; i < meshEntities.length; i++) { + meshEntities[i].present = false; + } + + if (data.meshesEnabled) { + for (var mesh of detectedMeshes.values()) { + // Ignore meshes that don't match the filterLabels. + if (filterLabels.length && filterLabels.indexOf(mesh.semanticLabel) === -1) { continue; } + for (i = 0; i < meshEntities.length; i++) { + if (mesh === meshEntities[i].mesh) { + present = true; + meshEntities[i].present = true; + if (meshEntities[i].lastChangedTime < mesh.lastChangedTime) { + this.updateMeshGeometry(meshEntities[i].el, mesh); + } + meshEntities[i].lastChangedTime = mesh.lastChangedTime; + break; + } + } + if (!present) { newMeshes.push(mesh); } + present = false; + } + } + + if (data.planesEnabled) { + for (mesh of detectedPlanes.values()) { + // Ignore meshes that don't match the filterLabels. + if (filterLabels.length && filterLabels.indexOf(mesh.semanticLabel) === -1) { continue; } + for (i = 0; i < meshEntities.length; i++) { + if (mesh === meshEntities[i].mesh) { + present = true; + meshEntities[i].present = true; + if (meshEntities[i].lastChangedTime < mesh.lastChangedTime) { + this.updateMeshGeometry(meshEntities[i].el, mesh); + } + meshEntities[i].lastChangedTime = mesh.lastChangedTime; + break; + } + } + if (!present) { newMeshes.push(mesh); } + present = false; + } + } + + this.deleteMeshes(); + this.createNewMeshes(newMeshes); + }, + + updateMeshes: (function () { + var auxMatrix = new THREE.Matrix4(); + return function () { + var meshPose; + var sceneEl = this.el; + var meshEl; + var frame = sceneEl.frame; + var meshEntities = this.meshEntities; + var referenceSpace = sceneEl.renderer.xr.getReferenceSpace(); + var meshSpace; + for (var i = 0; i < meshEntities.length; i++) { + meshSpace = meshEntities[i].mesh.meshSpace || meshEntities[i].mesh.planeSpace; + meshPose = frame.getPose(meshSpace, referenceSpace); + meshEl = meshEntities[i].el; + if (!meshEl.hasLoaded) { continue; } + auxMatrix.fromArray(meshPose.transform.matrix); + auxMatrix.decompose(meshEl.object3D.position, meshEl.object3D.quaternion, meshEl.object3D.scale); + } + }; + })(), + + deleteMeshes: function () { + var meshEntities = this.meshEntities; + var newMeshEntities = []; + for (var i = 0; i < meshEntities.length; i++) { + if (!meshEntities[i].present) { + this.el.removeChild(meshEntities[i]); + } else { + newMeshEntities.push(meshEntities[i]); + } + } + this.meshEntities = newMeshEntities; + }, + + createNewMeshes: function (newMeshes) { + var meshEl; + for (var i = 0; i < newMeshes.length; i++) { + meshEl = document.createElement('a-entity'); + this.meshEntities.push({ + mesh: newMeshes[i], + el: meshEl + }); + meshEl.addEventListener('loaded', this.initWorldMeshEntity); + this.el.appendChild(meshEl); + } + }, + + initMeshGeometry: function (mesh) { + var geometry; + var shape; + var polygon; + + if (mesh instanceof XRPlane) { + shape = new THREE.Shape(); + polygon = mesh.polygon; + for (var i = 0; i < polygon.length; ++i) { + if (i === 0) { + shape.moveTo(polygon[i].x, polygon[i].z); + } else { + shape.lineTo(polygon[i].x, polygon[i].z); + } + } + geometry = new THREE.ShapeGeometry(shape); + geometry.rotateX(Math.PI / 2); + return geometry; + } + + geometry = new THREE.BufferGeometry(); + geometry.setAttribute( + 'position', + new THREE.BufferAttribute(mesh.vertices, 3) + ); + geometry.setIndex(new THREE.BufferAttribute(mesh.indices, 1)); + return geometry; + }, + + initWorldMeshEntity: function (evt) { + var el = evt.target; + var geometry; + var mesh; + var meshEntity; + var meshEntities = this.meshEntities; + for (var i = 0; i < meshEntities.length; i++) { + if (meshEntities[i].el === el) { + meshEntity = meshEntities[i]; + break; + } + } + geometry = this.initMeshGeometry(meshEntity.mesh); + mesh = new THREE.Mesh(geometry, new THREE.MeshBasicMaterial({color: Math.random() * 0xFFFFFF, side: THREE.DoubleSide})); + el.setObject3D('mesh', mesh); + if (meshEntity.mesh instanceof XRPlane && this.data.planeMixin) { + el.setAttribute('mixin', this.data.planeMixin); + } else { + if (this.data.meshMixin) { + el.setAttribute('mixin', this.data.meshMixin); + } + } + el.setAttribute('data-world-mesh', meshEntity.mesh.semanticLabel); + }, + + updateMeshGeometry: function (entityEl, mesh) { + var entityMesh = entityEl.getObject3D('mesh'); + entityMesh.geometry.dispose(); + entityMesh.geometry = this.initMeshGeometry(mesh); + } +});