From 362dc352ee94352c17c370654ac3ac950f02219e Mon Sep 17 00:00:00 2001 From: AmoebaChant <10319625+AmoebaChant@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:24:12 -0700 Subject: [PATCH] FHL demo app (#51) --- .gitignore | 5 +- .vscode/settings.json | 3 +- package-lock.json | 31 +++++ package.json | 1 + packages/demo/package.json | 3 + packages/demo/src/fhl.ts | 62 +++++++++ packages/demo/src/fhl/effects/IEffect.ts | 9 ++ packages/demo/src/fhl/effects/likeEffect.ts | 53 ++++++++ packages/demo/src/fhl/reactionsSmartFilter.ts | 49 +++++++ packages/demo/src/fhl/smartFilterVideoApp.ts | 111 ++++++++++++++++ packages/demo/webpack.config.js | 3 +- packages/demo/www/fhl.html | 122 ++++++++++++++++++ 12 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 packages/demo/src/fhl.ts create mode 100644 packages/demo/src/fhl/effects/IEffect.ts create mode 100644 packages/demo/src/fhl/effects/likeEffect.ts create mode 100644 packages/demo/src/fhl/reactionsSmartFilter.ts create mode 100644 packages/demo/src/fhl/smartFilterVideoApp.ts create mode 100644 packages/demo/www/fhl.html diff --git a/.gitignore b/.gitignore index afb89ada..06fe6215 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,7 @@ dist !/src/smartFiltersEditor/components/log # generated shader TS files -**/*.shader.ts \ No newline at end of file +**/*.shader.ts + +# https certs +cert \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 6a342d8d..943bb132 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -31,7 +31,8 @@ "VANDENBERGHE", "webgl", "webgpu", - "glsl" + "glsl", + "SAMPLINGMODE" ], "files.exclude": { "**/node_modules": true, diff --git a/package-lock.json b/package-lock.json index 90c26969..6d6a7b2d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2986,6 +2986,16 @@ "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", "dev": true }, + "node_modules/@microsoft/teams-js": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/@microsoft/teams-js/-/teams-js-2.28.0.tgz", + "integrity": "sha512-WcQ2vWY+AvTOxToANe4znqDBvURDvZTztpjy5sYT24vsFimWKydOwT3OFE1/Av+gzofPrtqn+mf9bq6i9ED7Tg==", + "dev": true, + "dependencies": { + "base64-js": "^1.3.1", + "debug": "^4.3.3" + } + }, "node_modules/@microsoft/tsdoc": { "version": "0.14.2", "resolved": "https://registry.npmjs.org/@microsoft/tsdoc/-/tsdoc-0.14.2.tgz", @@ -4682,6 +4692,26 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -15936,6 +15966,7 @@ "@fortawesome/fontawesome-svg-core": "^6.1.0", "@fortawesome/free-solid-svg-icons": "^6.1.0", "@fortawesome/react-fontawesome": "^0.1.18", + "@microsoft/teams-js": "^2.19.0", "@types/react": "^17.0.30", "@types/react-dom": "^17.0.10", "css-loader": "^6.4.0", diff --git a/package.json b/package.json index 55f44946..dcb2cc5c 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "build:shaders": "npm run build:runTools -w @babylonjs/smart-filters && npm run build:runTools -w @babylonjs/smart-filters-demo", "watch": "tsc -b -v -w", "start": "npm run assets && concurrently \"npm run watch\" \"npm run start -w @babylonjs/smart-filters-demo\"", + "start:https": "npm run assets && concurrently \"npm run watch\" \"npm run start:https -w @babylonjs/smart-filters-demo\"", "test": "jest", "format": "npm run format:check && npm run format:fix", "format:check": "prettier --check \"./packages/**/src/**/*.{ts,tsx,js,json,scss,css}\"", diff --git a/packages/demo/package.json b/packages/demo/package.json index dc242883..6b4e8324 100644 --- a/packages/demo/package.json +++ b/packages/demo/package.json @@ -27,6 +27,8 @@ "watch:shaders": "node ../../node_modules/@babylonjs/smart-filters/dist/utils/buildTools/watchShaders.js ./src/configuration/blocks @babylonjs/smart-filters", "clean": "rimraf .temp && rimraf www/scripts", "start": "npm run watch", + "start:https": "concurrently \"npm run watch:https\" \"npm run watch:shaders\" \"npm run watch:shaders -w @babylonjs/smart-filters\"", + "watch:https": "npx webpack-dev-server --server-type https --server-options-key ./cert/smartFilters.test.key --server-options-cert ./cert/smartFilters.test.crt", "analyze": "webpack --profile --json > www/scripts/stats.json && npx webpack-bundle-analyzer www/scripts/stats.json", "preparePublish": "node build/preparePublish.js" }, @@ -36,6 +38,7 @@ "@fortawesome/fontawesome-svg-core": "^6.1.0", "@fortawesome/free-solid-svg-icons": "^6.1.0", "@fortawesome/react-fontawesome": "^0.1.18", + "@microsoft/teams-js": "^2.19.0", "react": "^17.0.2", "react-dom": "^17.0.2", "css-loader": "^6.4.0", diff --git a/packages/demo/src/fhl.ts b/packages/demo/src/fhl.ts new file mode 100644 index 00000000..a3e0a649 --- /dev/null +++ b/packages/demo/src/fhl.ts @@ -0,0 +1,62 @@ +import { app, videoEffects } from "@microsoft/teams-js"; +import { LOCAL_SMART_FILTER_EFFECT_ID, SMART_FILTER_EFFECT_ID, SmartFilterVideoApp } from "./fhl/smartFilterVideoApp"; +import { Observable } from "@babylonjs/core/Misc/observable"; + +// Read page elements +const likeButton = document.getElementById("likeButton") as HTMLButtonElement; +const outputCanvas = document.getElementById("outputCanvas") as HTMLCanvasElement; + +// Register button click handlers +const onLikeClickedObservable = new Observable(); +likeButton.addEventListener("click", () => { + onLikeClickedObservable.notifyObservers(); +}); + +// Initialize the SmartFilter Video App +console.log("Initializing SmartFilter Video App..."); +const videoApp = new SmartFilterVideoApp(outputCanvas, onLikeClickedObservable); + +/** + * Main function to initialize the app. + */ +async function main(): Promise { + console.log("Initializing SmartFilter Runtime..."); + await videoApp.initRuntime(); + + try { + console.log("Initializing Teams API..."); + await app.initialize(); + + console.log("Registering for video effect selections..."); + videoEffects.registerForVideoEffect(videoApp.videoEffectSelected.bind(videoApp)); + + console.log("Registering for video frame callbacks..."); + videoEffects.registerForVideoFrame({ + videoFrameHandler: videoApp.videoFrameHandler.bind(videoApp), + /** + * Callback function to process the video frames shared by the host. + */ + videoBufferHandler: videoApp.videoBufferHandler.bind(videoApp), + /** + * Video frame configuration supplied to the host to customize the generated video frame parameters, like format + */ + config: { + format: videoEffects.VideoFrameFormat.NV12, + }, + }); + + // Tell Teams we want to show our effect + videoEffects.notifySelectedVideoEffectChanged( + videoEffects.EffectChangeType.EffectChanged, + document.location.hostname.indexOf("localhost") !== -1 + ? LOCAL_SMART_FILTER_EFFECT_ID + : SMART_FILTER_EFFECT_ID + ); + } catch (e) { + console.log("Initialize failed - not in Teams - running in debug mode:", e); + } +} + +main().catch((e) => { + console.error(e); +}); diff --git a/packages/demo/src/fhl/effects/IEffect.ts b/packages/demo/src/fhl/effects/IEffect.ts new file mode 100644 index 00000000..6d18abbb --- /dev/null +++ b/packages/demo/src/fhl/effects/IEffect.ts @@ -0,0 +1,9 @@ +import type { Observable } from "@babylonjs/core/Misc/observable"; + +export interface IEffect { + readonly isStarted: boolean; + start(): void; + stop(notifyEffectCompleted: boolean): void; + + readonly onEffectCompleted: Observable; +} diff --git a/packages/demo/src/fhl/effects/likeEffect.ts b/packages/demo/src/fhl/effects/likeEffect.ts new file mode 100644 index 00000000..8381a48e --- /dev/null +++ b/packages/demo/src/fhl/effects/likeEffect.ts @@ -0,0 +1,53 @@ +import { Observable } from "@babylonjs/core/Misc/observable"; +import type { ReactionsSmartFilter } from "../reactionsSmartFilter"; +import type { IEffect } from "./IEffect"; +import type { Nullable } from "@babylonjs/core/types"; + +export class LikeEffect implements IEffect { + private _isStarted = false; + private _smartFilter: ReactionsSmartFilter; + + public onEffectCompleted: Observable; + private _timeout: Nullable = null; + + public get isStarted(): boolean { + return this._isStarted; + } + + public constructor(smartFilter: ReactionsSmartFilter) { + this._smartFilter = smartFilter; + this.onEffectCompleted = new Observable(); + } + + public start(): void { + if (this._isStarted) { + return; + } + this._isStarted = true; + + console.log("[LikeEffect] Starting"); + this._smartFilter.blackAndWhiteDisabled = false; + this._timeout = setTimeout(() => { + console.log("[LikeEffect] Timer ticked"); + this.stop(true); + }, 2000); + } + + public stop(notifyEffectCompleted: boolean): void { + if (!this._isStarted) { + return; + } + console.log("[LikeEffect] Stopping"); + this._isStarted = false; + + if (this._timeout) { + clearTimeout(this._timeout); + this._timeout = null; + } + this._smartFilter.blackAndWhiteDisabled = true; + + if (notifyEffectCompleted) { + this.onEffectCompleted.notifyObservers(); + } + } +} diff --git a/packages/demo/src/fhl/reactionsSmartFilter.ts b/packages/demo/src/fhl/reactionsSmartFilter.ts new file mode 100644 index 00000000..a9cc3691 --- /dev/null +++ b/packages/demo/src/fhl/reactionsSmartFilter.ts @@ -0,0 +1,49 @@ +import type { SmartFilterRuntime } from "@babylonjs/smart-filters"; +import { ConnectionPointType, createStrongRef, InputBlock, SmartFilter } from "@babylonjs/smart-filters"; +import type { ThinEngine } from "@babylonjs/core/Engines/thinEngine"; +import type { ThinTexture } from "@babylonjs/core/Materials/Textures/thinTexture"; +import { BlackAndWhiteBlock } from "../configuration/blocks/effects/blackAndWhiteBlock"; + +export class ReactionsSmartFilter { + private _blackAndWhiteDisabledInputBlock: InputBlock; + private _engine: ThinEngine; + public smartFilter: SmartFilter; + public textureInputBlock: InputBlock; + + public get blackAndWhiteDisabled(): boolean { + return this._blackAndWhiteDisabledInputBlock.runtimeValue.value; + } + public set blackAndWhiteDisabled(value: boolean) { + this._blackAndWhiteDisabledInputBlock.runtimeValue.value = value; + } + + constructor(engine: ThinEngine) { + this._engine = engine; + + this.smartFilter = new SmartFilter("Teams Reactions"); + this.textureInputBlock = new InputBlock( + this.smartFilter, + "videoFrame", + ConnectionPointType.Texture, + createStrongRef(null) + ); + this._blackAndWhiteDisabledInputBlock = new InputBlock( + this.smartFilter, + "blackAndWhiteDisabled", + ConnectionPointType.Boolean, + createStrongRef(true) + ); + const blackAndWhiteBlock = new BlackAndWhiteBlock(this.smartFilter, "BlackAndWhite"); + this._blackAndWhiteDisabledInputBlock.output.connectTo(blackAndWhiteBlock.disabled); + this.textureInputBlock.output.connectTo(blackAndWhiteBlock.input); + blackAndWhiteBlock.output.connectTo(this.smartFilter.output); + } + + public async initRuntime(inputTexture: ThinTexture): Promise { + const smartFilterRuntime = await this.smartFilter.createRuntimeAsync(this._engine); + + this.textureInputBlock.runtimeValue.value = inputTexture; + + return smartFilterRuntime; + } +} diff --git a/packages/demo/src/fhl/smartFilterVideoApp.ts b/packages/demo/src/fhl/smartFilterVideoApp.ts new file mode 100644 index 00000000..906dc6b9 --- /dev/null +++ b/packages/demo/src/fhl/smartFilterVideoApp.ts @@ -0,0 +1,111 @@ +import type { videoEffects } from "@microsoft/teams-js"; +import { ThinEngine } from "@babylonjs/core/Engines/thinEngine"; +import { Texture } from "@babylonjs/core/Materials/Textures/texture"; +import type { SmartFilterRuntime } from "@babylonjs/smart-filters"; +import type { Nullable } from "@babylonjs/core/types"; +import { ThinTexture } from "@babylonjs/core/Materials/Textures/thinTexture"; +import "@babylonjs/core/Engines/Extensions/engine.dynamicTexture"; +import type { InternalTexture } from "@babylonjs/core/Materials/Textures/internalTexture"; +import { ReactionsSmartFilter } from "./reactionsSmartFilter"; +import type { IEffect } from "./effects/IEffect"; +import { LikeEffect } from "./effects/likeEffect"; +import type { Observable } from "@babylonjs/core/Misc/observable"; + +export const SMART_FILTER_EFFECT_ID = "f71bd30b-c5e9-48ff-b039-42bc19df95a8"; +export const LOCAL_SMART_FILTER_EFFECT_ID = "fb9f0fab-9eb9-4756-8588-8dc3c6ad04d0"; + +export class SmartFilterVideoApp { + private _outputCanvas: HTMLCanvasElement; + private _onLikeClickedObservable: Observable; + + private _engine: ThinEngine; + private _internalInputTexture: InternalTexture; + private _inputTexture: ThinTexture; + private _smartFilter: ReactionsSmartFilter; + private _currentEffect: Nullable = null; + private _smartFilterRuntime: Nullable = null; + + constructor(outputCanvas: HTMLCanvasElement, onLikeClickedObservable: Observable) { + this._outputCanvas = outputCanvas; + this._onLikeClickedObservable = onLikeClickedObservable; + + this._engine = new ThinEngine(this._outputCanvas); + + // Create Dynamic Texture + this._internalInputTexture = this._engine.createDynamicTexture( + this._outputCanvas.width, + this._outputCanvas.height, + false, + Texture.BILINEAR_SAMPLINGMODE + ); + this._inputTexture = new ThinTexture(this._internalInputTexture); + + // Create Smart Filter + this._smartFilter = new ReactionsSmartFilter(this._engine); + } + + public async initRuntime(): Promise { + this._smartFilterRuntime = await this._smartFilter.initRuntime(this._inputTexture); + + // Listen to button clicks + this._onLikeClickedObservable.add(() => { + console.log("Like button clicked"); + this._startEffect(new LikeEffect(this._smartFilter)); + }); + } + + public videoEffectSelected(effectId: string | undefined): Promise { + console.log("videoEffectSelected() called", effectId); + return Promise.resolve(); + } + + private _startEffect(effect: IEffect): void { + if (this._currentEffect) { + this._currentEffect.stop(false); + } + + this._currentEffect = effect; + this._currentEffect.onEffectCompleted.add(() => { + console.log("Effect completed"); + if (this._currentEffect == effect) { + this._currentEffect = null; + } + }); + this._currentEffect.start(); + } + + async videoFrameHandler(frame: videoEffects.VideoFrameData): Promise { + try { + const videoFrame = frame.videoFrame as VideoFrame; + + if (this._currentEffect === null) { + return videoFrame; + } + + this._engine.updateDynamicTexture(this._internalInputTexture, videoFrame as unknown as any, true); + + this._engine.beginFrame(); + this._smartFilterRuntime!.render(); + this._engine.endFrame(); + + const outputVideoFrame = new VideoFrame(this._outputCanvas, { + displayHeight: videoFrame.displayHeight, + displayWidth: videoFrame.displayWidth, + timestamp: videoFrame.timestamp, + }); + + return outputVideoFrame; + } catch (e) { + console.error(e); + throw e; + } + } + + async videoBufferHandler( + _videoBufferData: videoEffects.VideoBufferData, + _notifyVideoFrameProcessed: () => void, + _notifyError: (error: string) => void + ) { + console.error("videoBufferHandler was called and is not yet implemented"); + } +} diff --git a/packages/demo/webpack.config.js b/packages/demo/webpack.config.js index a9ee98a1..bd463134 100644 --- a/packages/demo/webpack.config.js +++ b/packages/demo/webpack.config.js @@ -9,7 +9,8 @@ var buildConfig = function (env) { return { context: __dirname, entry: { - index: SRC_DIR + "/app.ts" + index: SRC_DIR + "/app.ts", + fhl: SRC_DIR + "/fhl.ts", }, performance: { maxEntrypointSize: 5120000, diff --git a/packages/demo/www/fhl.html b/packages/demo/www/fhl.html new file mode 100644 index 00000000..b1f853ff --- /dev/null +++ b/packages/demo/www/fhl.html @@ -0,0 +1,122 @@ + + + + + + + + Babylon.js Smart Filters - FHL September 2024 + + + +
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+ + + + + +