feat: cortex extension engine

core/src/browser/extension.ts

@@ -31,6 +31,13 @@ const ALL_INSTALLATION_STATE = [
 export type InstallationStateTuple = typeof ALL_INSTALLATION_STATE
 export type InstallationState = InstallationStateTuple[number]
 
+export type InstallationPackage = {
+  name: string;
+  version: string;
+  installationState: InstallationState;
+  description?: string;
+};
+
 /**
  * Represents a base extension.
  * This class should be extended by any class that represents an extension.
@@ -202,11 +209,25 @@ export abstract class BaseExtension implements ExtensionType {
 
     await fs.writeFileSync(settingPath, JSON.stringify(updatedSettings, null, 2))
 
-    updatedSettings.forEach((setting) => {
-      this.onSettingUpdate<typeof setting.controllerProps.value>(
+    await Promise.all(updatedSettings.map((setting) => {
+      this.onSettingUpdate<typeof setting.controllerProps.value | typeof setting.children>(
         setting.key,
-        setting.controllerProps.value
+        setting.children ? setting.children : setting.controllerProps.value
       )
-    })
+    }))
   }
+
+  installationPackages(): Promise<InstallationPackage[]>{
+    return Promise.resolve([])
+  }
+
+  installPackage(packageName: string): Promise<void>{
+    return Promise.resolve()
+  }
+
+  abortPackageInstallation(packageName: string): Promise<void>{
+    return Promise.resolve()
+   }
+
+
 }

core/src/browser/extensions/engines/AIEngine.ts

@@ -13,7 +13,7 @@ export abstract class AIEngine extends BaseExtension {
   private static modelsFolder = 'models'
 
   // The inference engine
-  abstract provider: string
+  abstract providers: string[]
 
   /**
    * On extension load, subscribe to events.
@@ -79,15 +79,19 @@ export abstract class AIEngine extends BaseExtension {
    * Loads the model.
    */
   async loadModel(model: Model): Promise<any> {
-    if (model.engine.toString() !== this.provider) return Promise.resolve()
+    if(!this.providers.includes(model.engine.toString())) {
+      return Promise.resolve()
+    }
     events.emit(ModelEvent.OnModelReady, model)
     return Promise.resolve()
   }
   /**
    * Stops the model.
    */
   async unloadModel(model?: Model): Promise<any> {
-    if (model?.engine && model.engine.toString() !== this.provider) return Promise.resolve()
+    if(model?.engine && !this.providers.includes(model.engine.toString())) {
+      return Promise.resolve()
+    }
     events.emit(ModelEvent.OnModelStopped, model ?? {})
     return Promise.resolve()
   }

core/src/browser/extensions/engines/EngineManager.ts

@@ -11,7 +11,7 @@ export class EngineManager {
    * @param engine - The engine to register.
    */
   register<T extends AIEngine>(engine: T) {
-    this.engines.set(engine.provider, engine)
+    engine.providers.forEach((provider) => this.engines.set(provider, engine))
   }
 
   /**

core/src/browser/extensions/engines/LocalOAIEngine.ts

@@ -27,7 +27,7 @@ export abstract class LocalOAIEngine extends OAIEngine {
    * Load the model.
    */
   override async loadModel(model: Model): Promise<void> {
-    if (model.engine.toString() !== this.provider) return
+    if(!this.providers.includes(model.engine.toString())) return
     const modelFolderName = 'models'
     const modelFolder = await joinPath([await getJanDataFolderPath(), modelFolderName, model.id])
     const systemInfo = await systemInformation()
@@ -54,7 +54,7 @@ export abstract class LocalOAIEngine extends OAIEngine {
    * Stops the model.
    */
   override async unloadModel(model?: Model) {
-    if (model?.engine && model.engine?.toString() !== this.provider) return Promise.resolve()
+    if(model?.engine && !this.providers.includes(model.engine.toString())) return Promise.resolve()
 
     this.loadedModel = undefined
     await executeOnMain(this.nodeModule, this.unloadModelFunctionName).then(() => {

core/src/browser/extensions/engines/OAIEngine.ts

@@ -4,6 +4,7 @@ import { AIEngine } from './AIEngine'
 import {
   ChatCompletionRole,
   ContentType,
+  InferenceEngine,
   InferenceEvent,
   MessageEvent,
   MessageRequest,
@@ -55,7 +56,10 @@ export abstract class OAIEngine extends AIEngine {
    * Inference request
    */
   override async inference(data: MessageRequest) {
-    if (data.model?.engine?.toString() !== this.provider) return
+    const isCortex = [InferenceEngine.cortex_llamacpp, InferenceEngine.cortex_onnx, InferenceEngine.cortex_tensorrtllm].includes(data.model?.engine as any)
+    const engine = data.model?.engine
+
+    if(!this.providers.includes(engine ?? '')) return
 
     const timestamp = Date.now()
     const message: ThreadMessage = {
@@ -80,7 +84,7 @@ export abstract class OAIEngine extends AIEngine {
 
     const model: ModelInfo = {
       ...(this.loadedModel ? this.loadedModel : {}),
-      ...data.model,
+      ...data.model as any,
     }
 
     const header = await this.headers()
@@ -89,6 +93,7 @@ export abstract class OAIEngine extends AIEngine {
       model: model.id,
       stream: true,
       ...model.parameters,
+      ...(isCortex ? { engine: data.model?.engine } : {}),
     }
     if (this.transformPayload) {
       requestBody = this.transformPayload(requestBody)

core/src/types/file/index.ts

@@ -20,7 +20,7 @@ export type DownloadState = {
   localPath?: string
 }
 
-export type DownloadType = 'model' | 'extension'
+export type DownloadType = 'model' | 'extension' | 'engine'
 
 export type DownloadRequest = {
   /**

core/src/types/setting/settingComponent.ts

@@ -4,15 +4,15 @@ export type SettingComponentProps = {
   description: string
   controllerType: ControllerType
   controllerProps: SliderComponentProps | CheckboxComponentProps | InputComponentProps
-
   extensionName?: string
   requireModelReload?: boolean
   configType?: ConfigType
+  children?: SettingComponentProps[]
 }
 
 export type ConfigType = 'runtime' | 'setting'
 
-export type ControllerType = 'slider' | 'checkbox' | 'input'
+export type ControllerType = 'slider' | 'checkbox' | 'input' | 'formGroup'
 
 export type InputType = 'password' | 'text' | 'email' | 'number' | 'tel' | 'url'
 

extensions/inference-anthropic-extension/src/index.ts

@@ -32,7 +32,7 @@ type AnthropicPayloadType = {
  */
 export default class JanInferenceAnthropicExtension extends RemoteOAIEngine {
   inferenceUrl: string = ''
-  provider: string = 'anthropic'
+  providers: string[] = ['anthropic']
   maxTokens: number = 4096
 
   override async onLoad(): Promise<void> {

extensions/inference-cohere-extension/src/index.ts

@@ -37,7 +37,7 @@ type CoherePayloadType = {
  */
 export default class JanInferenceCohereExtension extends RemoteOAIEngine {
   inferenceUrl: string = ''
-  provider: string = 'cohere'
+  providers: string[] = ['cohere']
 
   override async onLoad(): Promise<void> {
     super.onLoad()

extensions/inference-cortex-extension/package.json

@@ -27,6 +27,7 @@
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-replace": "^5.0.5",
     "@types/decompress": "^4.2.7",
+    "@types/eventsource": "^1.1.15",
     "@types/jest": "^29.5.12",
     "@types/node": "^20.11.4",
     "@types/os-utils": "^0.0.4",
@@ -36,6 +37,7 @@
     "jest": "^29.7.0",
     "rimraf": "^3.0.2",
     "rollup": "^2.38.5",
+    "rollup-plugin-copy": "^3.5.0",
     "rollup-plugin-define": "^1.0.1",
     "rollup-plugin-sourcemaps": "^0.6.3",
     "rollup-plugin-typescript2": "^0.36.0",
@@ -46,9 +48,16 @@
   "dependencies": {
     "@huggingface/hub": "^0.15.1",
     "@janhq/core": "file:../../core",
+    "@nestjs/microservices": "^10.4.1",
+    "@nestjs/websockets": "^10.4.1",
+    "class-transformer": "^0.5.1",
+    "cortexso": "^v0.5.0-47",
     "decompress": "^4.2.1",
+    "eventsource": "^2.0.2",
     "fetch-retry": "^5.0.6",
+    "pg-hstore": "^2.3.4",
     "rxjs": "^7.8.1",
+    "sqlite3": "^5.1.7",
     "tcp-port-used": "^1.0.2",
     "terminate": "2.6.1",
     "ulidx": "^2.3.0"
@@ -66,6 +75,12 @@
     "fetch-retry",
     "@janhq/core",
     "decompress",
-    "cortexso"
+    "cortexso",
+    "@nestjs/microservices",
+    "@nestjs/websockets",
+    "class-transformer",
+    "pg-hstore",
+    "rxjs",
+    "sqlite3"
   ]
 }

extensions/inference-cortex-extension/resources/default_settings.json

@@ -1,33 +1,44 @@
 [
   {
-    "key": "test",
-    "title": "Test",
-    "description": "Test",
-    "controllerType": "input",
+    "key": "cortex-process-config",
+    "title": "Cortex Process Configuration",
+    "description": "The configuration of the Cortex process.",
+    "controllerType": "formGroup",
     "controllerProps": {
-      "placeholder": "Test",
+      "placeholder": "",
       "value": ""
+    },
+    "children":
+  [{
+    "key": "cortex-host",
+    "title": "Cortex Host",
+    "description": "The host of the Cortex API.",
+    "controllerType": "input",
+    "controllerProps": {
+      "placeholder": "Input the host of the Cortex process",
+      "value": "127.0.0.1"
     }
   },
   {
-    "key": "embedding",
-    "title": "Embedding",
-    "description": "Whether to enable embedding.",
-    "controllerType": "checkbox",
+    "key": "cortex-port",
+    "title": "Cortex Port",
+    "description": "The port of the Cortex API.",
+    "controllerType": "input",
     "controllerProps": {
-      "value": true
+      "placeholder": "Input the port of the Cortex process",
+      "value": "1338"
     }
   },
   {
-    "key": "ctx_len",
-    "title": "Context Length",
-    "description": "The context length for model operations varies; the maximum depends on the specific model used.",
-    "controllerType": "slider",
+    "key": "cortex-engine-port",
+    "title": "Cortex Engine Port",
+    "description": "The port of the Cortex Engine.",
+    "controllerType": "input",
     "controllerProps": {
-      "min": 0,
-      "max": 4096,
-      "step": 128,
-      "value": 2048
+      "placeholder": "Input the port of the Cortex Engine",
+      "value": "3930"
     }
   }
+]
+  }
 ]

extensions/inference-cortex-extension/rollup.config.ts

@@ -25,7 +25,7 @@ export default async () => [
         DEFAULT_SETTINGS: JSON.stringify(defaultSettingJson),
         INFERENCE_URL: JSON.stringify(
           process.env.INFERENCE_URL ||
-            'http://127.0.0.1:3941/inferences/server/chat_completion'
+            'http://127.0.0.1:1338/v1/chat/completions',
         ),
         TROUBLESHOOTING_URL: JSON.stringify(
           'https://jan.ai/guides/troubleshooting'
@@ -53,27 +53,27 @@ export default async () => [
   {
     input: `src/node/index.ts`,
     output: [
-      { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: true },
+      { file: 'dist/node/index.cjs.js', format: 'cjs', sourcemap: false},
     ],
     // Indicate here external modules you don't wanna include in your bundle (i.e.: 'lodash')
-    external: ['@janhq/core/node'],
+    external: ['@janhq/core/node', 'cortexso', '@janhq/core'],
     watch: {
       include: 'src/node/**',
     },
+    inlineDynamicImports: true,
     plugins: [
       // Allow json resolution
       json(),
       // Compile TypeScript files
       typescript({ useTsconfigDeclarationDir: true }),
+      resolve({
+        extensions: ['.ts', '.js', '.json', '.mjs', '.js', '.json', '.node'],
+      }),
       // Allow bundling cjs modules (unlike webpack, rollup doesn't understand cjs)
       commonjs(),
       // Allow node_modules resolution, so you can use 'external' to control
       // which external modules to include in the bundle
       // https://github.com/rollup/rollup-plugin-node-resolve#usage
-      resolve({
-        extensions: ['.ts', '.js', '.json'],
-      }),
-
       // Resolve source maps to the original source
       sourceMaps(),
     ],