feat: support ar in webxr (#1665)

* feat: support ar in webxr

* feat: each tick on canvas will pass webxr frame

* chore: commit changeset

.changeset/witty-turtles-design.md

@@ -0,0 +1,7 @@
+---
+'@antv/g-plugin-device-renderer': patch
+'@antv/g-webgl': patch
+'@antv/g-lite': patch
+---
+
+Pass webxr frame on each tick when rendering.

__tests__/demos/3d/index.ts

@@ -3,3 +3,4 @@ export { sphere } from './sphere';
 export { torus } from './torus';
 export { cylinder } from './cylinder';
 export { force } from './force';
+export { ar } from './webar';

__tests__/demos/3d/webar.ts

@@ -0,0 +1,64 @@
+import { CanvasEvent } from '../../../packages/g';
+import {
+  MeshBasicMaterial,
+  CubeGeometry,
+  Mesh,
+  Plugin as Plugin3D,
+} from '../../../packages/g-plugin-3d';
+import { Plugin as PluginControl } from '../../../packages/g-plugin-control';
+import { ARButton, DeviceRenderer } from '../../../packages/g-webgl';
+
+export async function ar(context) {
+  const { canvas, renderer, container } = context;
+
+  // wait for canvas' initialization complete
+  await canvas.ready;
+
+  // use GPU device
+  const plugin = renderer.getPlugin('device-renderer') as DeviceRenderer.Plugin;
+  const device = plugin.getDevice();
+
+  // 1. load texture with URL
+  const map = plugin.loadTexture(
+    'https://gw.alipayobjects.com/mdn/rms_6ae20b/afts/img/A*_aqoS73Se3sAAAAAAAAAAAAAARQnAQ',
+  );
+
+  const cubeGeometry = new CubeGeometry(device, {
+    width: 200,
+    height: 200,
+    depth: 200,
+  });
+  const basicMaterial = new MeshBasicMaterial(device, {
+    // wireframe: true,
+    map,
+  });
+
+  const cube = new Mesh({
+    style: {
+      fill: '#1890FF',
+      opacity: 1,
+      geometry: cubeGeometry,
+      material: basicMaterial,
+    },
+  });
+
+  cube.setPosition(300, 250, 200);
+
+  canvas.appendChild(cube);
+
+  canvas.addEventListener(CanvasEvent.AFTER_RENDER, () => {
+    cube.rotate(1, 0, 0);
+  });
+
+  canvas.getConfig().disableHitTesting = true;
+
+  const $button = ARButton.createButton(canvas, renderer, {});
+  container.appendChild($button);
+}
+
+ar.initRenderer = (renderer, type) => {
+  if (type === 'webgl' || type === 'webgpu') {
+    renderer.registerPlugin(new Plugin3D());
+    renderer.registerPlugin(new PluginControl());
+  }
+};

__tests__/main.ts

@@ -189,6 +189,7 @@ function createSpecRender(object) {
         ],
         // Used for WebGPU renderer
         shaderCompilerPath: '/glsl_wgsl_compiler_bg.wasm',
+        // enableAutoRendering: false,
         // enableDirtyRectangleRendering: false,
         // enableDirtyRectangleRenderingDebug: true,
       });

packages/g-lite/package.json

@@ -53,7 +53,8 @@
     "@types/d3-color": "^3.0.2",
     "@types/gl-matrix": "^2.4.5",
     "@types/offscreencanvas": "^2019.6.4",
-    "@types/rbush": "^3.0.0"
+    "@types/rbush": "^3.0.0",
+    "@types/webxr": "0.5.5"
   },
   "publishConfig": {
     "access": "public"

packages/g-lite/src/Canvas.ts

@@ -481,11 +481,11 @@ export class Canvas extends EventTarget implements ICanvas {
     this.document.documentElement.destroyChildren();
   }
 
-  render() {
+  render(frame?: XRFrame) {
     this.dispatchEvent(beforeRenderEvent);
 
     const renderingService = this.getRenderingService();
-    renderingService.render(this.getConfig(), () => {
+    renderingService.render(this.getConfig(), frame, () => {
       // trigger actual rerender event
       // @see https://github.com/antvis/G/issues/1268
       this.dispatchEvent(rerenderEvent);
@@ -495,11 +495,11 @@ export class Canvas extends EventTarget implements ICanvas {
   }
 
   private run() {
-    const tick = () => {
-      this.render();
+    const tick = (time: number, frame?: XRFrame) => {
+      this.render(frame);
       this.frameId = this.requestAnimationFrame(tick);
     };
-    tick();
+    tick(0);
   }
 
   private initRenderer(renderer: IRenderer, firstContentfullPaint = false) {

packages/g-lite/src/camera/Camera.ts

@@ -305,6 +305,13 @@ export class Camera implements ICamera {
     return this;
   }
 
+  /**
+   * Set projection matrix manually.
+   */
+  setProjectionMatrix(matrix: mat4) {
+    this.projectionMatrix = matrix;
+  }
+
   setFov(fov: number) {
     this.setPerspective(this.near, this.far, fov, this.aspect);
     return this;

packages/g-lite/src/services/RenderingService.ts

@@ -84,7 +84,7 @@ export class RenderingService {
     /**
      * called at beginning of each frame, won't get called if nothing to re-render
      */
-    beginFrame: new SyncHook<[]>(),
+    beginFrame: new SyncHook<[XRFrame]>(),
     /**
      * called before every dirty object get rendered
      */
@@ -97,7 +97,7 @@ export class RenderingService {
      * called after every dirty object get rendered
      */
     afterRender: new SyncHook<[DisplayObject]>(),
-    endFrame: new SyncHook<[]>(),
+    endFrame: new SyncHook<[XRFrame]>(),
     destroy: new SyncHook<[]>(),
     /**
      * use async but faster method such as GPU-based picking in `g-plugin-device-renderer`
@@ -161,7 +161,11 @@ export class RenderingService {
     );
   }
 
-  render(canvasConfig: Partial<CanvasConfig>, rerenderCallback: () => void) {
+  render(
+    canvasConfig: Partial<CanvasConfig>,
+    frame: XRFrame,
+    rerenderCallback: () => void,
+  ) {
     this.stats.total = 0;
     this.stats.rendered = 0;
     this.zIndexCounter = 0;
@@ -191,7 +195,7 @@ export class RenderingService {
         );
       }
 
-      this.hooks.beginFrame.call();
+      this.hooks.beginFrame.call(frame);
 
       if (shouldTriggerRenderHooks) {
         renderingContext.renderListCurrentFrame.forEach((object) => {
@@ -201,7 +205,7 @@ export class RenderingService {
         });
       }
 
-      this.hooks.endFrame.call();
+      this.hooks.endFrame.call(frame);
       renderingContext.renderListCurrentFrame = [];
       renderingContext.renderReasons.clear();
 

packages/g-plugin-device-renderer/package.json

@@ -54,6 +54,7 @@
     "@rollup/plugin-wasm": "^5.1.2",
     "@types/earcut": "^2.1.1",
     "@types/offscreencanvas": "^2019.6.4",
+    "@types/webxr": "0.5.5",
     "glslify-import": "3.1.0",
     "rollup-plugin-glslify": "^1.2.1"
   },