add support for exporting to json and csv in workspace chats (#622)

* add support for exporting to json and csv in workspace chats

* safety encode URL options

* remove message about openai fine tuning on export success

* all defaults to jsonl

frontend/src/index.css

@@ -407,3 +407,35 @@ dialog::backdrop {
 .Toastify__toast-body {
   white-space: pre-line;
 }
+
+@keyframes slideDown {
+  from {
+    max-height: 0;
+    opacity: 0;
+  }
+
+  to {
+    max-height: 400px;
+    opacity: 1;
+  }
+}
+
+.slide-down {
+  animation: slideDown 0.3s ease-out forwards;
+}
+
+@keyframes slideUp {
+  from {
+    max-height: 400px;
+    opacity: 1;
+  }
+
+  to {
+    max-height: 0;
+    opacity: 0;
+  }
+}
+
+.slide-up {
+  animation: slideUp 0.3s ease-out forwards;
+}

frontend/src/models/system.js

@@ -398,8 +398,10 @@ const System = {
         return { success: false, error: e.message };
       });
   },
-  exportChats: async () => {
-    return await fetch(`${API_BASE}/system/export-chats`, {
+  exportChats: async (type = "csv") => {
+    const url = new URL(`${fullApiUrl()}/system/export-chats`);
+    url.searchParams.append("type", encodeURIComponent(type));
+    return await fetch(url, {
       method: "GET",
       headers: baseHeaders(),
     })

frontend/src/pages/GeneralSettings/Chats/index.jsx

@@ -1,4 +1,4 @@
-import { useEffect, useState } from "react";
+import { useEffect, useRef, useState } from "react";
 import Sidebar, { SidebarMobileHeader } from "@/components/SettingsSidebar";
 import { isMobile } from "react-device-detect";
 import * as Skeleton from "react-loading-skeleton";
@@ -7,29 +7,60 @@ import useQuery from "@/hooks/useQuery";
 import ChatRow from "./ChatRow";
 import showToast from "@/utils/toast";
 import System from "@/models/system";
-
-const PAGE_SIZE = 20;
+import { CaretDown } from "@phosphor-icons/react";
 export default function WorkspaceChats() {
+  const [showMenu, setShowMenu] = useState(false);
+  const [exportType, setExportType] = useState("jsonl");
+  const menuRef = useRef();
+  const openMenuButton = useRef();
+
+  const exportOptions = {
+    csv: { mimeType: "text/csv", fileExtension: "csv" },
+    json: { mimeType: "application/json", fileExtension: "json" },
+    jsonl: { mimeType: "application/jsonl", fileExtension: "jsonl" },
+  };
   const handleDumpChats = async () => {
-    const chats = await System.exportChats();
+    const chats = await System.exportChats(exportType);
     if (chats) {
-      const blob = new Blob([chats], { type: "application/jsonl" });
+      const { mimeType, fileExtension } = exportOptions[exportType];
+      const blob = new Blob([chats], { type: mimeType });
       const link = document.createElement("a");
       link.href = window.URL.createObjectURL(blob);
-      link.download = "chats.jsonl";
+      link.download = `chats.${fileExtension}`;
       document.body.appendChild(link);
       link.click();
       window.URL.revokeObjectURL(link.href);
       document.body.removeChild(link);
       showToast(
-        "Chats exported successfully. Note: Must have at least 10 chats to be valid for OpenAI fine tuning.",
+        `Chats exported successfully as ${fileExtension.toUpperCase()}.`,
         "success"
       );
     } else {
       showToast("Failed to export chats.", "error");
     }
   };
 
+  const toggleMenu = () => {
+    setShowMenu(!showMenu);
+  };
+
+  useEffect(() => {
+    function handleClickOutside(event) {
+      if (
+        menuRef.current &&
+        !menuRef.current.contains(event.target) &&
+        !openMenuButton.current.contains(event.target)
+      ) {
+        setShowMenu(false);
+      }
+    }
+
+    document.addEventListener("mousedown", handleClickOutside);
+    return () => {
+      document.removeEventListener("mousedown", handleClickOutside);
+    };
+  }, []);
+
   return (
     <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
       {!isMobile && <Sidebar />}
@@ -44,12 +75,46 @@ export default function WorkspaceChats() {
               <p className="text-2xl font-semibold text-white">
                 Workspace Chats
               </p>
-              <button
-                onClick={handleDumpChats}
-                className="border border-slate-200 px-4 py-1 rounded-lg text-slate-200 text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800"
-              >
-                Export Chats to JSONL
-              </button>
+              <div className="flex gap-x-1 relative">
+                <button
+                  onClick={handleDumpChats}
+                  className="border border-slate-200 px-4 py-1 rounded-lg text-slate-200 text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800"
+                >
+                  Export Chats to {exportType.toUpperCase()}
+                </button>
+                <button
+                  ref={openMenuButton}
+                  onClick={toggleMenu}
+                  className={`transition-all duration-300 border border-slate-200 p-1 rounded-lg text-slate-200 text-sm items-center flex hover:bg-slate-200 hover:text-slate-800 ${
+                    showMenu ? "bg-slate-200 text-slate-800" : ""
+                  }`}
+                >
+                  <CaretDown weight="bold" className="h-4 w-4" />
+                </button>
+                <div
+                  ref={menuRef}
+                  className={`${
+                    showMenu ? "slide-down" : "slide-up hidden"
+                  } z-20 w-fit rounded-lg absolute top-full right-0 bg-sidebar p-4 flex items-center justify-center mt-2`}
+                >
+                  <div className="flex flex-col gap-y-2">
+                    {Object.keys(exportOptions)
+                      .filter((type) => type !== exportType)
+                      .map((type) => (
+                        <button
+                          key={type}
+                          onClick={() => {
+                            setExportType(type);
+                            setShowMenu(false);
+                          }}
+                          className="text-white hover:bg-slate-200/20 w-full text-left px-4 py-1.5 rounded-md"
+                        >
+                          {type.toUpperCase()}
+                        </button>
+                      ))}
+                  </div>
+                </div>
+              </div>
             </div>
             <p className="text-sm font-base text-white text-opacity-60">
               These are all the recorded chats and messages that have been sent

server/endpoints/system.js

@@ -47,6 +47,7 @@ const { WorkspaceChats } = require("../models/workspaceChats");
 const { Workspace } = require("../models/workspace");
 const { flexUserRoleValid } = require("../utils/middleware/multiUserProtected");
 const { fetchPfp, determinePfpFilepath } = require("../utils/files/pfp");
+const { convertToCSV, convertToJSON, convertToJSONL } = require("./utils");
 
 function systemEndpoints(app) {
   if (!app) return;
@@ -776,8 +777,9 @@ function systemEndpoints(app) {
   app.get(
     "/system/export-chats",
     [validatedRequest, flexUserRoleValid],
-    async (_request, response) => {
+    async (request, response) => {
       try {
+        const { type = "jsonl" } = request.query;
         const chats = await WorkspaceChats.whereWithData({}, null, null, {
           id: "asc",
         });
@@ -828,13 +830,27 @@ function systemEndpoints(app) {
           return acc;
         }, {});
 
-        // Convert to JSONL
-        const jsonl = Object.values(workspaceChatsMap)
-          .map((workspaceChats) => JSON.stringify(workspaceChats))
-          .join("\n");
+        let output;
+        switch (type.toLowerCase()) {
+          case "json": {
+            response.setHeader("Content-Type", "application/json");
+            output = await convertToJSON(workspaceChatsMap);
+            break;
+          }
+          case "csv": {
+            response.setHeader("Content-Type", "text/csv");
+            output = await convertToCSV(workspaceChatsMap);
+            break;
+          }
+          // JSONL default
+          default: {
+            response.setHeader("Content-Type", "application/jsonl");
+            output = await convertToJSONL(workspaceChatsMap);
+            break;
+          }
+        }
 
-        response.setHeader("Content-Type", "application/jsonl");
-        response.status(200).send(jsonl);
+        response.status(200).send(output);
       } catch (e) {
         console.error(e);
         response.sendStatus(500).end();

server/endpoints/utils.js

@@ -32,6 +32,34 @@ async function getDiskStorage() {
   }
 }
 
+async function convertToCSV(workspaceChatsMap) {
+  const rows = ["role,content"];
+  for (const workspaceChats of Object.values(workspaceChatsMap)) {
+    for (const message of workspaceChats.messages) {
+      // Escape double quotes and wrap content in double quotes
+      const escapedContent = `"${message.content
+        .replace(/"/g, '""')
+        .replace(/\n/g, " ")}"`;
+      rows.push(`${message.role},${escapedContent}`);
+    }
+  }
+  return rows.join("\n");
+}
+
+async function convertToJSON(workspaceChatsMap) {
+  const allMessages = [].concat.apply(
+    [],
+    Object.values(workspaceChatsMap).map((workspace) => workspace.messages)
+  );
+  return JSON.stringify(allMessages);
+}
+
+async function convertToJSONL(workspaceChatsMap) {
+  return Object.values(workspaceChatsMap)
+    .map((workspaceChats) => JSON.stringify(workspaceChats))
+    .join("\n");
+}
+
 function utilEndpoints(app) {
   if (!app) return;
 
@@ -54,4 +82,10 @@ function utilEndpoints(app) {
   });
 }
 
-module.exports = { utilEndpoints, getGitVersion };
+module.exports = {
+  utilEndpoints,
+  getGitVersion,
+  convertToCSV,
+  convertToJSON,
+  convertToJSONL,
+};