FHL demo app (#51)

.gitignore

@@ -33,4 +33,7 @@ dist
 !/src/smartFiltersEditor/components/log
 
 # generated shader TS files
-**/*.shader.ts
+**/*.shader.ts
+
+# https certs
+cert

.vscode/settings.json

@@ -31,7 +31,8 @@
         "VANDENBERGHE",
         "webgl",
         "webgpu",
-        "glsl"
+        "glsl",
+        "SAMPLINGMODE"
     ],
     "files.exclude": {
         "**/node_modules": true,

package.json

@@ -29,6 +29,7 @@
         "build:shaders": "npm run build:runTools -w @babylonjs/smart-filters && npm run build:runTools -w @babylonjs/smart-filters-demo",
         "watch": "tsc -b -v -w",
         "start": "npm run assets && concurrently \"npm run watch\" \"npm run start -w @babylonjs/smart-filters-demo\"",
+        "start:https": "npm run assets && concurrently \"npm run watch\" \"npm run start:https -w @babylonjs/smart-filters-demo\"",
         "test": "jest",
         "format": "npm run format:check && npm run format:fix",
         "format:check": "prettier --check \"./packages/**/src/**/*.{ts,tsx,js,json,scss,css}\"",

packages/demo/package.json

@@ -27,6 +27,8 @@
         "watch:shaders": "node ../../node_modules/@babylonjs/smart-filters/dist/utils/buildTools/watchShaders.js ./src/configuration/blocks @babylonjs/smart-filters",
         "clean": "rimraf .temp && rimraf www/scripts",
         "start": "npm run watch",
+        "start:https": "concurrently \"npm run watch:https\" \"npm run watch:shaders\" \"npm run watch:shaders -w @babylonjs/smart-filters\"",
+        "watch:https": "npx webpack-dev-server --server-type https --server-options-key ./cert/smartFilters.test.key --server-options-cert ./cert/smartFilters.test.crt",
         "analyze": "webpack --profile --json > www/scripts/stats.json && npx webpack-bundle-analyzer www/scripts/stats.json",
         "preparePublish": "node build/preparePublish.js"
     },
@@ -36,6 +38,7 @@
         "@fortawesome/fontawesome-svg-core": "^6.1.0",
         "@fortawesome/free-solid-svg-icons": "^6.1.0",
         "@fortawesome/react-fontawesome": "^0.1.18",
+        "@microsoft/teams-js": "^2.19.0",
         "react": "^17.0.2",
         "react-dom": "^17.0.2",
         "css-loader": "^6.4.0",

packages/demo/src/fhl.ts

@@ -0,0 +1,62 @@
+import { app, videoEffects } from "@microsoft/teams-js";
+import { LOCAL_SMART_FILTER_EFFECT_ID, SMART_FILTER_EFFECT_ID, SmartFilterVideoApp } from "./fhl/smartFilterVideoApp";
+import { Observable } from "@babylonjs/core/Misc/observable";
+
+// Read page elements
+const likeButton = document.getElementById("likeButton") as HTMLButtonElement;
+const outputCanvas = document.getElementById("outputCanvas") as HTMLCanvasElement;
+
+// Register button click handlers
+const onLikeClickedObservable = new Observable<void>();
+likeButton.addEventListener("click", () => {
+    onLikeClickedObservable.notifyObservers();
+});
+
+// Initialize the SmartFilter Video App
+console.log("Initializing SmartFilter Video App...");
+const videoApp = new SmartFilterVideoApp(outputCanvas, onLikeClickedObservable);
+
+/**
+ * Main function to initialize the app.
+ */
+async function main(): Promise<void> {
+    console.log("Initializing SmartFilter Runtime...");
+    await videoApp.initRuntime();
+
+    try {
+        console.log("Initializing Teams API...");
+        await app.initialize();
+
+        console.log("Registering for video effect selections...");
+        videoEffects.registerForVideoEffect(videoApp.videoEffectSelected.bind(videoApp));
+
+        console.log("Registering for video frame callbacks...");
+        videoEffects.registerForVideoFrame({
+            videoFrameHandler: videoApp.videoFrameHandler.bind(videoApp),
+            /**
+             * Callback function to process the video frames shared by the host.
+             */
+            videoBufferHandler: videoApp.videoBufferHandler.bind(videoApp),
+            /**
+             * Video frame configuration supplied to the host to customize the generated video frame parameters, like format
+             */
+            config: {
+                format: videoEffects.VideoFrameFormat.NV12,
+            },
+        });
+
+        // Tell Teams we want to show our effect
+        videoEffects.notifySelectedVideoEffectChanged(
+            videoEffects.EffectChangeType.EffectChanged,
+            document.location.hostname.indexOf("localhost") !== -1
+                ? LOCAL_SMART_FILTER_EFFECT_ID
+                : SMART_FILTER_EFFECT_ID
+        );
+    } catch (e) {
+        console.log("Initialize failed - not in Teams - running in debug mode:", e);
+    }
+}
+
+main().catch((e) => {
+    console.error(e);
+});

packages/demo/src/fhl/effects/IEffect.ts

@@ -0,0 +1,9 @@
+import type { Observable } from "@babylonjs/core/Misc/observable";
+
+export interface IEffect {
+    readonly isStarted: boolean;
+    start(): void;
+    stop(notifyEffectCompleted: boolean): void;
+
+    readonly onEffectCompleted: Observable<void>;
+}

packages/demo/src/fhl/effects/likeEffect.ts

@@ -0,0 +1,53 @@
+import { Observable } from "@babylonjs/core/Misc/observable";
+import type { ReactionsSmartFilter } from "../reactionsSmartFilter";
+import type { IEffect } from "./IEffect";
+import type { Nullable } from "@babylonjs/core/types";
+
+export class LikeEffect implements IEffect {
+    private _isStarted = false;
+    private _smartFilter: ReactionsSmartFilter;
+
+    public onEffectCompleted: Observable<void>;
+    private _timeout: Nullable<NodeJS.Timeout> = null;
+
+    public get isStarted(): boolean {
+        return this._isStarted;
+    }
+
+    public constructor(smartFilter: ReactionsSmartFilter) {
+        this._smartFilter = smartFilter;
+        this.onEffectCompleted = new Observable<void>();
+    }
+
+    public start(): void {
+        if (this._isStarted) {
+            return;
+        }
+        this._isStarted = true;
+
+        console.log("[LikeEffect] Starting");
+        this._smartFilter.blackAndWhiteDisabled = false;
+        this._timeout = setTimeout(() => {
+            console.log("[LikeEffect] Timer ticked");
+            this.stop(true);
+        }, 2000);
+    }
+
+    public stop(notifyEffectCompleted: boolean): void {
+        if (!this._isStarted) {
+            return;
+        }
+        console.log("[LikeEffect] Stopping");
+        this._isStarted = false;
+
+        if (this._timeout) {
+            clearTimeout(this._timeout);
+            this._timeout = null;
+        }
+        this._smartFilter.blackAndWhiteDisabled = true;
+
+        if (notifyEffectCompleted) {
+            this.onEffectCompleted.notifyObservers();
+        }
+    }
+}

packages/demo/src/fhl/reactionsSmartFilter.ts

@@ -0,0 +1,49 @@
+import type { SmartFilterRuntime } from "@babylonjs/smart-filters";
+import { ConnectionPointType, createStrongRef, InputBlock, SmartFilter } from "@babylonjs/smart-filters";
+import type { ThinEngine } from "@babylonjs/core/Engines/thinEngine";
+import type { ThinTexture } from "@babylonjs/core/Materials/Textures/thinTexture";
+import { BlackAndWhiteBlock } from "../configuration/blocks/effects/blackAndWhiteBlock";
+
+export class ReactionsSmartFilter {
+    private _blackAndWhiteDisabledInputBlock: InputBlock<ConnectionPointType.Boolean>;
+    private _engine: ThinEngine;
+    public smartFilter: SmartFilter;
+    public textureInputBlock: InputBlock<ConnectionPointType.Texture>;
+
+    public get blackAndWhiteDisabled(): boolean {
+        return this._blackAndWhiteDisabledInputBlock.runtimeValue.value;
+    }
+    public set blackAndWhiteDisabled(value: boolean) {
+        this._blackAndWhiteDisabledInputBlock.runtimeValue.value = value;
+    }
+
+    constructor(engine: ThinEngine) {
+        this._engine = engine;
+
+        this.smartFilter = new SmartFilter("Teams Reactions");
+        this.textureInputBlock = new InputBlock<ConnectionPointType.Texture>(
+            this.smartFilter,
+            "videoFrame",
+            ConnectionPointType.Texture,
+            createStrongRef(null)
+        );
+        this._blackAndWhiteDisabledInputBlock = new InputBlock<ConnectionPointType.Boolean>(
+            this.smartFilter,
+            "blackAndWhiteDisabled",
+            ConnectionPointType.Boolean,
+            createStrongRef(true)
+        );
+        const blackAndWhiteBlock = new BlackAndWhiteBlock(this.smartFilter, "BlackAndWhite");
+        this._blackAndWhiteDisabledInputBlock.output.connectTo(blackAndWhiteBlock.disabled);
+        this.textureInputBlock.output.connectTo(blackAndWhiteBlock.input);
+        blackAndWhiteBlock.output.connectTo(this.smartFilter.output);
+    }
+
+    public async initRuntime(inputTexture: ThinTexture): Promise<SmartFilterRuntime> {
+        const smartFilterRuntime = await this.smartFilter.createRuntimeAsync(this._engine);
+
+        this.textureInputBlock.runtimeValue.value = inputTexture;
+
+        return smartFilterRuntime;
+    }
+}

packages/demo/src/fhl/smartFilterVideoApp.ts

@@ -0,0 +1,111 @@
+import type { videoEffects } from "@microsoft/teams-js";
+import { ThinEngine } from "@babylonjs/core/Engines/thinEngine";
+import { Texture } from "@babylonjs/core/Materials/Textures/texture";
+import type { SmartFilterRuntime } from "@babylonjs/smart-filters";
+import type { Nullable } from "@babylonjs/core/types";
+import { ThinTexture } from "@babylonjs/core/Materials/Textures/thinTexture";
+import "@babylonjs/core/Engines/Extensions/engine.dynamicTexture";
+import type { InternalTexture } from "@babylonjs/core/Materials/Textures/internalTexture";
+import { ReactionsSmartFilter } from "./reactionsSmartFilter";
+import type { IEffect } from "./effects/IEffect";
+import { LikeEffect } from "./effects/likeEffect";
+import type { Observable } from "@babylonjs/core/Misc/observable";
+
+export const SMART_FILTER_EFFECT_ID = "f71bd30b-c5e9-48ff-b039-42bc19df95a8";
+export const LOCAL_SMART_FILTER_EFFECT_ID = "fb9f0fab-9eb9-4756-8588-8dc3c6ad04d0";
+
+export class SmartFilterVideoApp {
+    private _outputCanvas: HTMLCanvasElement;
+    private _onLikeClickedObservable: Observable<void>;
+
+    private _engine: ThinEngine;
+    private _internalInputTexture: InternalTexture;
+    private _inputTexture: ThinTexture;
+    private _smartFilter: ReactionsSmartFilter;
+    private _currentEffect: Nullable<IEffect> = null;
+    private _smartFilterRuntime: Nullable<SmartFilterRuntime> = null;
+
+    constructor(outputCanvas: HTMLCanvasElement, onLikeClickedObservable: Observable<void>) {
+        this._outputCanvas = outputCanvas;
+        this._onLikeClickedObservable = onLikeClickedObservable;
+
+        this._engine = new ThinEngine(this._outputCanvas);
+
+        // Create Dynamic Texture
+        this._internalInputTexture = this._engine.createDynamicTexture(
+            this._outputCanvas.width,
+            this._outputCanvas.height,
+            false,
+            Texture.BILINEAR_SAMPLINGMODE
+        );
+        this._inputTexture = new ThinTexture(this._internalInputTexture);
+
+        // Create Smart Filter
+        this._smartFilter = new ReactionsSmartFilter(this._engine);
+    }
+
+    public async initRuntime(): Promise<void> {
+        this._smartFilterRuntime = await this._smartFilter.initRuntime(this._inputTexture);
+
+        // Listen to button clicks
+        this._onLikeClickedObservable.add(() => {
+            console.log("Like button clicked");
+            this._startEffect(new LikeEffect(this._smartFilter));
+        });
+    }
+
+    public videoEffectSelected(effectId: string | undefined): Promise<void> {
+        console.log("videoEffectSelected() called", effectId);
+        return Promise.resolve();
+    }
+
+    private _startEffect(effect: IEffect): void {
+        if (this._currentEffect) {
+            this._currentEffect.stop(false);
+        }
+
+        this._currentEffect = effect;
+        this._currentEffect.onEffectCompleted.add(() => {
+            console.log("Effect completed");
+            if (this._currentEffect == effect) {
+                this._currentEffect = null;
+            }
+        });
+        this._currentEffect.start();
+    }
+
+    async videoFrameHandler(frame: videoEffects.VideoFrameData): Promise<VideoFrame> {
+        try {
+            const videoFrame = frame.videoFrame as VideoFrame;
+
+            if (this._currentEffect === null) {
+                return videoFrame;
+            }
+
+            this._engine.updateDynamicTexture(this._internalInputTexture, videoFrame as unknown as any, true);
+
+            this._engine.beginFrame();
+            this._smartFilterRuntime!.render();
+            this._engine.endFrame();
+
+            const outputVideoFrame = new VideoFrame(this._outputCanvas, {
+                displayHeight: videoFrame.displayHeight,
+                displayWidth: videoFrame.displayWidth,
+                timestamp: videoFrame.timestamp,
+            });
+
+            return outputVideoFrame;
+        } catch (e) {
+            console.error(e);
+            throw e;
+        }
+    }
+
+    async videoBufferHandler(
+        _videoBufferData: videoEffects.VideoBufferData,
+        _notifyVideoFrameProcessed: () => void,
+        _notifyError: (error: string) => void
+    ) {
+        console.error("videoBufferHandler was called and is not yet implemented");
+    }
+}

packages/demo/webpack.config.js

@@ -9,7 +9,8 @@ var buildConfig = function (env) {
     return {
         context: __dirname,
         entry: {
-            index: SRC_DIR + "/app.ts"
+            index: SRC_DIR + "/app.ts",
+            fhl: SRC_DIR + "/fhl.ts",
         },
         performance: {
             maxEntrypointSize: 5120000,