Merge pull request #189 from lobehub/feat/transformer

fix: 修复滚动问题 & 舞蹈动作播放问题

next.config.mjs

@@ -53,6 +53,12 @@ const nextConfig = {
       },
     );
 
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      'sharp$': false,
+      'onnxruntime-node$': false,
+    };
+
     return config;
   },
 };

package.json

@@ -97,6 +97,7 @@
     "@trpc/server": "next",
     "@types/react-speech-recognition": "^3.9.5",
     "@vercel/analytics": "^1.4.1",
+    "@xenova/transformers": "^2.17.2",
     "ahooks": "^3.8.4",
     "ai": "^2.2.37",
     "antd": "~5.18.3",

src/app/chat/ChatMode/SideBar/ChatHeader/index.tsx

@@ -16,6 +16,7 @@ export const useStyles = createStyles(({ css, token }) => ({
     position: sticky;
     inset-block-start: 0;
     height: 64px;
+    line-height: 64px;
   `,
 }));
 

src/app/chat/ChatMode/SideBar/SessionList/index.tsx

@@ -18,6 +18,7 @@ const List = dynamic(() => import('./List'), {
 
 const useStyles = createStyles(({ css, token, prefixCls }) => ({
   session: css`
+    overflow-y: auto;
     height: 100%;
   `,
   list: css`

src/app/transformers/page.tsx

@@ -0,0 +1,72 @@
+'use client';
+
+import { useCallback, useEffect, useRef, useState } from 'react';
+
+export default function Home() {
+  // Keep track of the classification result and the model loading status.
+  const [result, setResult] = useState(null);
+  const [ready, setReady] = useState(null);
+
+  // Create a reference to the worker object.
+  const worker = useRef(null);
+
+  // We use the `useEffect` hook to set up the worker as soon as the `App` component is mounted.
+  useEffect(() => {
+    if (!worker.current) {
+      // Create the worker if it does not yet exist.
+      worker.current = new Worker(new URL('../../workers/classification.ts', import.meta.url), {
+        type: 'module',
+      });
+    }
+
+    // Create a callback function for messages from the worker thread.
+    const onMessageReceived = (e) => {
+      switch (e.data.status) {
+        case 'initiate': {
+          setReady(false);
+          break;
+        }
+        case 'ready': {
+          setReady(true);
+          break;
+        }
+        case 'complete': {
+          setResult(e.data.output[0]);
+          break;
+        }
+      }
+    };
+
+    // Attach the callback function as an event listener.
+    worker.current.addEventListener('message', onMessageReceived);
+
+    // Define a cleanup function for when the component is unmounted.
+    return () => worker.current.removeEventListener('message', onMessageReceived);
+  });
+
+  const classify = useCallback((text) => {
+    if (worker.current) {
+      worker.current.postMessage({ text });
+    }
+  }, []);
+  return (
+    <main className="flex min-h-screen flex-col items-center justify-center p-12">
+      <h1 className="text-5xl font-bold mb-2 text-center">Transformers.js</h1>
+      <h2 className="text-2xl mb-4 text-center">Next.js template (client-side)</h2>
+      <input
+        type="text"
+        className="w-full max-w-xs p-2 border border-gray-300 rounded mb-4"
+        placeholder="Enter text here"
+        onChange={(e) => {
+          classify(e.target.value);
+        }}
+      />
+
+      {ready !== null && (
+        <pre className="bg-gray-100 p-2 rounded">
+          {!ready || !result ? 'Loading...' : JSON.stringify(result, null, 2)}
+        </pre>
+      )}
+    </main>
+  );
+}

src/libs/VMDAnimation/loadVMDCamera.ts

@@ -1,4 +1,4 @@
-import { AnimationClip, PerspectiveCamera } from 'three';
+import { AnimationClip, Box3, Object3D, PerspectiveCamera, Vector3 } from 'three';
 import { MMDAnimationHelper } from 'three/examples/jsm/animation/MMDAnimationHelper.js';
 import { MMDLoader } from 'three/examples/jsm/loaders/MMDLoader.js';
 

src/libs/vrmViewer/viewer.ts

@@ -16,6 +16,7 @@ import { MMDAnimationHelper } from 'three/examples/jsm/animation/MMDAnimationHel
 import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js';
 import { MMDLoader } from 'three/examples/jsm/loaders/MMDLoader.js';
 
+import { VRM_TO_MMD_SCALE } from '@/constants/common';
 import { loadVMDCamera } from '@/libs/VMDAnimation/loadVMDCamera';
 import { MotionFileType } from '@/libs/emoteController/type';
 import { TouchAreaEnum } from '@/types/touch';
@@ -105,7 +106,8 @@ export class Viewer {
     // 加载摄像机动画
     let cameraPromise = null;
     if (cameraUrl && this._camera) {
-      cameraPromise = loadVMDCamera(cameraUrl, this._camera).then((cameraAnimation) => {
+      const scale = this.calculateVRMScale();
+      cameraPromise = loadVMDCamera(cameraUrl, this._camera, scale).then((cameraAnimation) => {
         if (this._camera && cameraAnimation) {
           this._cameraMixer = new THREE.AnimationMixer(this._camera);
           this._cameraAction = this._cameraMixer.clipAction(cameraAnimation);
@@ -508,4 +510,23 @@ export class Viewer {
       this._camera.updateProjectionMatrix();
     }
   }
+
+  /**
+   * 计算当前加载的 VRM 模型的缩放比例
+   * @returns 基于模型高度计算的缩放比例
+   */
+  private calculateVRMScale(): number {
+    if (!this.model?.vrm) {
+      return VRM_TO_MMD_SCALE;
+    }
+
+    const boundingBox = new Box3().setFromObject(this.model.vrm.scene);
+    const size = new Vector3();
+    boundingBox.getSize(size);
+    const vrmHeight = size.y;
+
+    // MMD 标准模型高度约为 20 单位
+    const MMD_STANDARD_HEIGHT = 20;
+    return vrmHeight / MMD_STANDARD_HEIGHT;
+  }
 }

src/store/session/index.ts

@@ -205,7 +205,7 @@ export const createSessionStore: StateCreator<SessionStore, [['zustand/devtools'
             chatConfig: DEFAULT_CHAT_CONFIG,
           },
         };
-        draft.push(session);
+        draft.unshift(session);
       }
     });
 

src/workers/classification.ts

@@ -0,0 +1,33 @@
+import { pipeline } from '@xenova/transformers';
+
+// Use the Singleton pattern to enable lazy construction of the pipeline.
+const PipelineSingleton = {
+  task: 'text-classification',
+  model: 'Xenova/distilbert-base-uncased-finetuned-sst-2-english',
+  instance: null,
+
+  async getInstance(progress_callback = null) {
+    this.instance ??= await pipeline(this.task, this.model, { progress_callback });
+    return this.instance;
+  },
+};
+
+// Listen for messages from the main thread
+self.addEventListener('message', async (event) => {
+  // Retrieve the classification pipeline. When called for the first time,
+  // this will load the pipeline and save it for future use.
+  const classifier = await PipelineSingleton.getInstance((x) => {
+    // We also add a progress callback to the pipeline so that we can
+    // track model loading.
+    self.postMessage(x);
+  });
+
+  // Actually perform the classification
+  const output = await classifier(event.data.text);
+
+  // Send the output back to the main thread
+  self.postMessage({
+    status: 'complete',
+    output: output,
+  });
+});