fix: chat using web interface (#1889)

Signed-off-by: James <[email protected]>
Co-authored-by: James <[email protected]>

extensions/inference-nitro-extension/rollup.config.ts

@@ -27,6 +27,9 @@ export default [
         TROUBLESHOOTING_URL: JSON.stringify(
           "https://jan.ai/guides/troubleshooting"
         ),
+        JAN_SERVER_INFERENCE_URL: JSON.stringify(
+          "http://localhost:1337/v1/chat/completions"
+        ),
       }),
       // Allow json resolution
       json(),

extensions/inference-nitro-extension/src/@types/global.d.ts

@@ -1,6 +1,7 @@
 declare const NODE: string;
 declare const INFERENCE_URL: string;
 declare const TROUBLESHOOTING_URL: string;
+declare const JAN_SERVER_INFERENCE_URL: string;
 
 /**
  * The response from the initModel function.

extensions/inference-nitro-extension/src/helpers/sse.ts

@@ -6,6 +6,7 @@ import { Observable } from "rxjs";
  * @returns An Observable that emits the generated response as a string.
  */
 export function requestInference(
+  inferenceUrl: string,
   recentMessages: any[],
   model: Model,
   controller?: AbortController
@@ -17,7 +18,7 @@ export function requestInference(
       stream: true,
       ...model.parameters,
     });
-    fetch(INFERENCE_URL, {
+    fetch(inferenceUrl, {
       method: "POST",
       headers: {
         "Content-Type": "application/json",

extensions/inference-nitro-extension/src/index.ts

@@ -68,35 +68,48 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
    */
   private nitroProcessInfo: any = undefined;
 
+  private inferenceUrl = "";
+
   /**
    * Subscribes to events emitted by the @janhq/core package.
    */
   async onLoad() {
     if (!(await fs.existsSync(JanInferenceNitroExtension._homeDir))) {
-      await fs
-        .mkdirSync(JanInferenceNitroExtension._homeDir)
-        .catch((err: Error) => console.debug(err));
+      try {
+        await fs.mkdirSync(JanInferenceNitroExtension._homeDir);
+      } catch (e) {
+        console.debug(e);
+      }
+    }
+
+    // init inference url
+    // @ts-ignore
+    const electronApi = window?.electronAPI;
+    this.inferenceUrl = INFERENCE_URL;
+    if (!electronApi) {
+      this.inferenceUrl = JAN_SERVER_INFERENCE_URL;
     }
+    console.debug("Inference url: ", this.inferenceUrl);
 
     if (!(await fs.existsSync(JanInferenceNitroExtension._settingsDir)))
       await fs.mkdirSync(JanInferenceNitroExtension._settingsDir);
     this.writeDefaultEngineSettings();
 
     // Events subscription
     events.on(MessageEvent.OnMessageSent, (data: MessageRequest) =>
-      this.onMessageRequest(data),
+      this.onMessageRequest(data)
     );
 
     events.on(ModelEvent.OnModelInit, (model: Model) =>
-      this.onModelInit(model),
+      this.onModelInit(model)
     );
 
     events.on(ModelEvent.OnModelStop, (model: Model) =>
-      this.onModelStop(model),
+      this.onModelStop(model)
     );
 
     events.on(InferenceEvent.OnInferenceStopped, () =>
-      this.onInferenceStopped(),
+      this.onInferenceStopped()
     );
 
     // Attempt to fetch nvidia info
@@ -121,7 +134,7 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
       } else {
         await fs.writeFileSync(
           engineFile,
-          JSON.stringify(this._engineSettings, null, 2),
+          JSON.stringify(this._engineSettings, null, 2)
         );
       }
     } catch (err) {
@@ -149,7 +162,7 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
 
     this.getNitroProcesHealthIntervalId = setInterval(
       () => this.periodicallyGetNitroHealth(),
-      JanInferenceNitroExtension._intervalHealthCheck,
+      JanInferenceNitroExtension._intervalHealthCheck
     );
   }
 
@@ -206,7 +219,11 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
     return new Promise(async (resolve, reject) => {
       if (!this._currentModel) return Promise.reject("No model loaded");
 
-      requestInference(data.messages ?? [], this._currentModel).subscribe({
+      requestInference(
+        this.inferenceUrl,
+        data.messages ?? [],
+        this._currentModel
+      ).subscribe({
         next: (_content: any) => {},
         complete: async () => {
           resolve(message);
@@ -254,7 +271,12 @@ export default class JanInferenceNitroExtension extends InferenceExtension {
       ...(this._currentModel || {}),
       ...(data.model || {}),
     };
-    requestInference(data.messages ?? [], model, this.controller).subscribe({
+    requestInference(
+      this.inferenceUrl,
+      data.messages ?? [],
+      model,
+      this.controller
+    ).subscribe({
       next: (content: any) => {
         const messageContent: ThreadContent = {
           type: ContentType.Text,

web/containers/Toast/index.tsx

@@ -19,8 +19,8 @@ const ErrorIcon = () => {
       xmlns="http://www.w3.org/2000/svg"
     >
       <path
-        fill-rule="evenodd"
-        clip-rule="evenodd"
+        fillRule="evenodd"
+        clipRule="evenodd"
         d="M20 10C20 15.5228 15.5228 20 10 20H0.993697C0.110179 20 -0.332289 18.9229 0.292453 18.2929L2.2495 16.3195C0.843343 14.597 1.21409e-08 12.397 1.21409e-08 10C1.21409e-08 4.47715 4.47715 0 10 0C15.5228 0 20 4.47715 20 10ZM13.2071 6.79289C13.5976 7.18342 13.5976 7.81658 13.2071 8.20711L11.4142 10L13.2071 11.7929C13.5976 12.1834 13.5976 12.8166 13.2071 13.2071C12.8166 13.5976 12.1834 13.5976 11.7929 13.2071L10 11.4142L8.20711 13.2071C7.81658 13.5976 7.18342 13.5976 6.79289 13.2071C6.40237 12.8166 6.40237 12.1834 6.79289 11.7929L8.58579 10L6.79289 8.20711C6.40237 7.81658 6.40237 7.18342 6.79289 6.79289C7.18342 6.40237 7.81658 6.40237 8.20711 6.79289L10 8.58579L11.7929 6.79289C12.1834 6.40237 12.8166 6.40237 13.2071 6.79289Z"
         fill="#EA2E4E"
       />
@@ -38,8 +38,8 @@ const WarningIcon = () => {
       xmlns="http://www.w3.org/2000/svg"
     >
       <path
-        fill-rule="evenodd"
-        clip-rule="evenodd"
+        fillRule="evenodd"
+        clipRule="evenodd"
         d="M20 10C20 15.5228 15.5228 20 10 20H0.993697C0.110179 20 -0.332289 18.9229 0.292453 18.2929L2.2495 16.3195C0.843343 14.597 1.21409e-08 12.397 1.21409e-08 10C1.21409e-08 4.47715 4.47715 0 10 0C15.5228 0 20 4.47715 20 10ZM10.99 6C10.99 5.44772 10.5446 5 9.99502 5C9.44549 5 9 5.44772 9 6V10C9 10.5523 9.44549 11 9.99502 11C10.5446 11 10.99 10.5523 10.99 10V6ZM9.99502 13C9.44549 13 9 13.4477 9 14C9 14.5523 9.44549 15 9.99502 15H10.005C10.5545 15 11 14.5523 11 14C11 13.4477 10.5545 13 10.005 13H9.99502Z"
         fill="#FACC15"
       />
@@ -57,8 +57,8 @@ const SuccessIcon = () => {
       xmlns="http://www.w3.org/2000/svg"
     >
       <path
-        fill-rule="evenodd"
-        clip-rule="evenodd"
+        fillRule="evenodd"
+        clipRule="evenodd"
         d="M20 10C20 15.5228 15.5228 20 10 20H0.993697C0.110179 20 -0.332289 18.9229 0.292453 18.2929L2.2495 16.3195C0.843343 14.597 1.21409e-08 12.397 1.21409e-08 10C1.21409e-08 4.47715 4.47715 0 10 0C15.5228 0 20 4.47715 20 10ZM13.7071 8.70711C14.0976 8.31658 14.0976 7.68342 13.7071 7.29289C13.3166 6.90237 12.6834 6.90237 12.2929 7.29289L9 10.5858L7.70711 9.2929C7.31658 8.90237 6.68342 8.90237 6.29289 9.2929C5.90237 9.68342 5.90237 10.3166 6.29289 10.7071L8.29289 12.7071C8.48043 12.8946 8.73478 13 9 13C9.26522 13 9.51957 12.8946 9.70711 12.7071L13.7071 8.70711Z"
         fill="#34D399"
       />
@@ -76,8 +76,8 @@ const DefaultIcon = () => {
       xmlns="http://www.w3.org/2000/svg"
     >
       <path
-        fill-rule="evenodd"
-        clip-rule="evenodd"
+        fillRule="evenodd"
+        clipRule="evenodd"
         d="M10 20C15.5228 20 20 15.5228 20 10C20 4.47715 15.5228 0 10 0C4.47715 0 2.11188e-08 4.47715 2.11188e-08 10C2.11188e-08 12.397 0.843343 14.597 2.2495 16.3195L0.292453 18.2929C-0.332289 18.9229 0.110179 20 0.993697 20H10ZM5.5 8C5.5 7.44772 5.94772 7 6.5 7H13.5C14.0523 7 14.5 7.44772 14.5 8C14.5 8.55229 14.0523 9 13.5 9H6.5C5.94772 9 5.5 8.55229 5.5 8ZM6.5 11C5.94772 11 5.5 11.4477 5.5 12C5.5 12.5523 5.94772 13 6.5 13H9.5C10.0523 13 10.5 12.5523 10.5 12C10.5 11.4477 10.0523 11 9.5 11H6.5Z"
         fill="#60A5FA"
       />