Merge branch 'main' into refactor/openai-moderation

danny-avila · Dec 14, 2024 · 14c974d · 14c974d
2 parents e8dffd3 + f150355
commit 14c974d
Show file tree

Hide file tree

Showing 269 changed files with 6,802 additions and 2,564 deletions.
diff --git a/.env.example b/.env.example
@@ -119,6 +119,7 @@ BINGAI_TOKEN=user_provided
 # BEDROCK_AWS_DEFAULT_REGION=us-east-1 # A default region must be provided
 # BEDROCK_AWS_ACCESS_KEY_ID=someAccessKey
 # BEDROCK_AWS_SECRET_ACCESS_KEY=someSecretAccessKey
+# BEDROCK_AWS_SESSION_TOKEN=someSessionToken
 
 # Note: This example list is not meant to be exhaustive. If omitted, all known, supported model IDs will be included for you.
 # BEDROCK_AWS_MODELS=anthropic.claude-3-5-sonnet-20240620-v1:0,meta.llama3-1-8b-instruct-v1:0
@@ -140,7 +141,7 @@ GOOGLE_KEY=user_provided
 # GOOGLE_REVERSE_PROXY=
 
 # Gemini API (AI Studio)
-# GOOGLE_MODELS=gemini-exp-1121,gemini-exp-1114,gemini-1.5-flash-latest,gemini-1.0-pro,gemini-1.0-pro-001,gemini-1.0-pro-latest,gemini-1.0-pro-vision-latest,gemini-1.5-pro-latest,gemini-pro,gemini-pro-vision
+# GOOGLE_MODELS=gemini-2.0-flash-exp,gemini-exp-1121,gemini-exp-1114,gemini-1.5-flash-latest,gemini-1.0-pro,gemini-1.0-pro-001,gemini-1.0-pro-latest,gemini-1.0-pro-vision-latest,gemini-1.5-pro-latest,gemini-pro,gemini-pro-vision
 
 # Vertex AI
 # GOOGLE_MODELS=gemini-1.5-flash-preview-0514,gemini-1.5-pro-preview-0514,gemini-1.0-pro-vision-001,gemini-1.0-pro-002,gemini-1.0-pro-001,gemini-pro-vision,gemini-1.0-pro
@@ -177,10 +178,10 @@ OPENAI_API_KEY=user_provided
 DEBUG_OPENAI=false
 
 # TITLE_CONVO=false
-# OPENAI_TITLE_MODEL=gpt-3.5-turbo
+# OPENAI_TITLE_MODEL=gpt-4o-mini
 
 # OPENAI_SUMMARIZE=true
-# OPENAI_SUMMARY_MODEL=gpt-3.5-turbo
+# OPENAI_SUMMARY_MODEL=gpt-4o-mini
 
 # OPENAI_FORCE_PROMPT=true
 

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -10,7 +10,8 @@
       "env": {
         "NODE_ENV": "production"
       },
-      "console": "integratedTerminal"
+      "console": "integratedTerminal",
+      "envFile": "${workspaceFolder}/.env"
     }
   ]
 }
diff --git a/README.md b/README.md
@@ -38,42 +38,75 @@
 </a>
 </p>
 
-# 📃 Features
-
-- 🖥️ UI matching ChatGPT, including Dark mode, Streaming, and latest updates
-- 🤖 AI model selection:
-  - Anthropic (Claude), AWS Bedrock, OpenAI, Azure OpenAI, BingAI, ChatGPT, Google Vertex AI, Plugins, Assistants API (including Azure Assistants)
-- ✅ Compatible across both **[Remote & Local AI services](https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints):**
-  - groq, Ollama, Cohere, Mistral AI, Apple MLX, koboldcpp, OpenRouter, together.ai, Perplexity, ShuttleAI, and more
-- 🪄 Generative UI with **[Code Artifacts](https://youtu.be/GfTj7O4gmd0?si=WJbdnemZpJzBrJo3)**
-   - Create React, HTML code, and Mermaid diagrams right in chat
-- 💾 Create, Save, & Share Custom Presets
-- 🔀 Switch between AI Endpoints and Presets, mid-chat
-- 🔄 Edit, Resubmit, and Continue Messages with Conversation branching
-- 🌿 Fork Messages & Conversations for Advanced Context control
-- 💬 Multimodal Chat:
-    - Upload and analyze images with Claude 3, GPT-4 (including `gpt-4o` and `gpt-4o-mini`), and Gemini Vision 📸
-    - Chat with Files using Custom Endpoints, OpenAI, Azure, Anthropic, & Google. 🗃️
-    - Advanced Agents with Files, Code Interpreter, Tools, and API Actions 🔦
-      - Available through the [OpenAI Assistants API](https://platform.openai.com/docs/assistants/overview) 🌤️
-      - Non-OpenAI Agents in Active Development 🚧
-- 🌎 Multilingual UI:
-  - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro,
+# ✨ Features
+
+- 🖥️ **UI & Experience** inspired by ChatGPT with enhanced design and features
+
+- 🤖 **AI Model Selection**:  
+  - Anthropic (Claude), AWS Bedrock, OpenAI, Azure OpenAI, Google, Vertex AI, OpenAI Assistants API (incl. Azure)
+  - [Custom Endpoints](https://www.librechat.ai/docs/quick_start/custom_endpoints): Use any OpenAI-compatible API with LibreChat, no proxy required
+  - Compatible with [Local & Remote AI Providers](https://www.librechat.ai/docs/configuration/librechat_yaml/ai_endpoints):
+    - Ollama, groq, Cohere, Mistral AI, Apple MLX, koboldcpp, together.ai,
+    - OpenRouter, Perplexity, ShuttleAI, Deepseek, Qwen, and more
+
+- 🔧 **[Code Interpreter API](https://www.librechat.ai/docs/features/code_interpreter)**: 
+  - Secure, Sandboxed Execution in Python, Node.js (JS/TS), Go, C/C++, Java, PHP, Rust, and Fortran
+  - Seamless File Handling: Upload, process, and download files directly
+  - No Privacy Concerns: Fully isolated and secure execution
+
+- 🔦 **Agents & Tools Integration**:  
+  - **[LibreChat Agents](https://www.librechat.ai/docs/features/agents)**:
+    - No-Code Custom Assistants: Build specialized, AI-driven helpers without coding  
+    - Flexible & Extensible: Attach tools like DALL-E-3, file search, code execution, and more  
+    - Compatible with Custom Endpoints, OpenAI, Azure, Anthropic, AWS Bedrock, and more
+  - Use LibreChat Agents and OpenAI Assistants with Files, Code Interpreter, Tools, and API Actions
+
+- 🪄 **Generative UI with Code Artifacts**:  
+  - [Code Artifacts](https://youtu.be/GfTj7O4gmd0?si=WJbdnemZpJzBrJo3) allow creation of React, HTML, and Mermaid diagrams directly in chat
+
+- 💾 **Presets & Context Management**:  
+  - Create, Save, & Share Custom Presets  
+  - Switch between AI Endpoints and Presets mid-chat
+  - Edit, Resubmit, and Continue Messages with Conversation branching  
+  - [Fork Messages & Conversations](https://www.librechat.ai/docs/features/fork) for Advanced Context control
+
+- 💬 **Multimodal & File Interactions**:  
+  - Upload and analyze images with Claude 3, GPT-4o, o1, Llama-Vision, and Gemini 📸  
+  - Chat with Files using Custom Endpoints, OpenAI, Azure, Anthropic, AWS Bedrock, & Google 🗃️
+
+- 🌎 **Multilingual UI**:  
+  - English, 中文, Deutsch, Español, Français, Italiano, Polski, Português Brasileiro
   - Русский, 日本語, Svenska, 한국어, Tiếng Việt, 繁體中文, العربية, Türkçe, Nederlands, עברית
-- 🎨 Customizable Dropdown & Interface: Adapts to both power users and newcomers
-- 📧 Verify your email to ensure secure access
-- 🗣️ Chat hands-free with Speech-to-Text and Text-to-Speech magic
-  - Automatically send and play Audio
+
+- 🎨 **Customizable Interface**:  
+  - Customizable Dropdown & Interface that adapts to both power users and newcomers
+
+- 📧 **Secure Access**:  
+  - Verify your email to ensure secure access
+
+- 🗣️ **Speech & Audio**:  
+  - Chat hands-free with Speech-to-Text and Text-to-Speech  
+  - Automatically send and play Audio  
   - Supports OpenAI, Azure OpenAI, and Elevenlabs
-- 📥 Import Conversations from LibreChat, ChatGPT, Chatbot UI
-- 📤 Export conversations as screenshots, markdown, text, json
-- 🔍 Search all messages/conversations
-- 🔌 Plugins, including web access, image generation with DALL-E-3 and more
-- 👥 Multi-User, Secure Authentication with Moderation and Token spend tools
-- ⚙️ Configure Proxy, Reverse Proxy, Docker, & many Deployment options:
+
+- 📥 **Import & Export Conversations**:  
+  - Import Conversations from LibreChat, ChatGPT, Chatbot UI  
+  - Export conversations as screenshots, markdown, text, json
+
+- 🔍 **Search & Discovery**:  
+  - Search all messages/conversations
+
+- 👥 **Multi-User & Secure**:
+  - Multi-User, Secure Authentication with OAuth2 & Email Login Support
+  - Built-in Moderation, and Token spend tools
+
+- ⚙️ **Configuration & Deployment**:  
+  - Configure Proxy, Reverse Proxy, Docker, & many Deployment options  
   - Use completely local or deploy on the cloud
-- 📖 Completely Open-Source & Built in Public
-- 🧑‍🤝‍🧑 Community-driven development, support, and feedback
+
+- 📖 **Open-Source & Community**:  
+  - Completely Open-Source & Built in Public  
+  - Community-driven development, support, and feedback
 
 [For a thorough review of our features, see our docs here](https://docs.librechat.ai/) 📚
 

diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js
@@ -50,6 +50,8 @@ class BaseClient {
     /** The key for the usage object's output tokens
      * @type {string} */
     this.outputTokensKey = 'completion_tokens';
+    /** @type {Set<string>} */
+    this.savedMessageIds = new Set();
   }
 
   setOptions() {
@@ -84,7 +86,7 @@ class BaseClient {
       return this.options.agent.id;
     }
 
-    return this.modelOptions.model;
+    return this.modelOptions?.model ?? this.model;
   }
 
   /**
@@ -508,7 +510,7 @@ class BaseClient {
           conversationId,
           parentMessageId: userMessage.messageId,
           isCreatedByUser: false,
-          model: this.modelOptions.model,
+          model: this.modelOptions?.model ?? this.model,
           sender: this.sender,
           text: generation,
         };
@@ -545,6 +547,7 @@ class BaseClient {
 
     if (!isEdited && !this.skipSaveUserMessage) {
       this.userMessagePromise = this.saveMessageToDatabase(userMessage, saveOptions, user);
+      this.savedMessageIds.add(userMessage.messageId);
       if (typeof opts?.getReqData === 'function') {
         opts.getReqData({
           userMessagePromise: this.userMessagePromise,
@@ -563,8 +566,8 @@ class BaseClient {
           user: this.user,
           tokenType: 'prompt',
           amount: promptTokens,
-          model: this.modelOptions.model,
           endpoint: this.options.endpoint,
+          model: this.modelOptions?.model ?? this.model,
           endpointTokenConfig: this.options.endpointTokenConfig,
         },
       });
@@ -574,6 +577,7 @@ class BaseClient {
     const completion = await this.sendCompletion(payload, opts);
     this.abortController.requestCompleted = true;
 
+    /** @type {TMessage} */
     const responseMessage = {
       messageId: responseMessageId,
       conversationId,
@@ -635,7 +639,16 @@ class BaseClient {
       responseMessage.attachments = (await Promise.all(this.artifactPromises)).filter((a) => a);
     }
 
+    if (this.options.attachments) {
+      try {
+        saveOptions.files = this.options.attachments.map((attachments) => attachments.file_id);
+      } catch (error) {
+        logger.error('[BaseClient] Error mapping attachments for conversation', error);
+      }
+    }
+
     this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user);
+    this.savedMessageIds.add(responseMessage.messageId);
     const messageCache = getLogStores(CacheKeys.MESSAGES);
     messageCache.set(
       responseMessageId,
@@ -902,8 +915,9 @@ class BaseClient {
     // Note: gpt-3.5-turbo and gpt-4 may update over time. Use default for these as well as for unknown models
     let tokensPerMessage = 3;
     let tokensPerName = 1;
+    const model = this.modelOptions?.model ?? this.model;
 
-    if (this.modelOptions.model === 'gpt-3.5-turbo-0301') {
+    if (model === 'gpt-3.5-turbo-0301') {
       tokensPerMessage = 4;
       tokensPerName = -1;
     }
@@ -961,6 +975,15 @@ class BaseClient {
       return _messages;
     }
 
+    const seen = new Set();
+    const attachmentsProcessed =
+      this.options.attachments && !(this.options.attachments instanceof Promise);
+    if (attachmentsProcessed) {
+      for (const attachment of this.options.attachments) {
+        seen.add(attachment.file_id);
+      }
+    }
+
     /**
      *
      * @param {TMessage} message
@@ -971,7 +994,19 @@ class BaseClient {
         this.message_file_map = {};
       }
 
-      const fileIds = message.files.map((file) => file.file_id);
+      const fileIds = [];
+      for (const file of message.files) {
+        if (seen.has(file.file_id)) {
+          continue;
+        }
+        fileIds.push(file.file_id);
+        seen.add(file.file_id);
+      }
+
+      if (fileIds.length === 0) {
+        return message;
+      }
+
       const files = await getFiles({
         file_id: { $in: fileIds },
       });

diff --git a/api/app/clients/ChatGPTClient.js b/api/app/clients/ChatGPTClient.js
@@ -227,6 +227,16 @@ class ChatGPTClient extends BaseClient {
       this.azure = !serverless && azureOptions;
       this.azureEndpoint =
         !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+      if (serverless === true) {
+        this.options.defaultQuery = azureOptions.azureOpenAIApiVersion
+          ? { 'api-version': azureOptions.azureOpenAIApiVersion }
+          : undefined;
+        this.options.headers['api-key'] = this.apiKey;
+      }
+    }
+
+    if (this.options.defaultQuery) {
+      opts.defaultQuery = this.options.defaultQuery;
     }
 
     if (this.options.headers) {

diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js
@@ -688,7 +688,7 @@ class OpenAIClient extends BaseClient {
   }
 
   initializeLLM({
-    model = 'gpt-3.5-turbo',
+    model = 'gpt-4o-mini',
     modelName,
     temperature = 0.2,
     presence_penalty = 0,
@@ -793,7 +793,7 @@ class OpenAIClient extends BaseClient {
 
     const { OPENAI_TITLE_MODEL } = process.env ?? {};
 
-    let model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-3.5-turbo';
+    let model = this.options.titleModel ?? OPENAI_TITLE_MODEL ?? 'gpt-4o-mini';
     if (model === Constants.CURRENT_MODEL) {
       model = this.modelOptions.model;
     }
@@ -838,6 +838,12 @@ class OpenAIClient extends BaseClient {
       this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
       this.options.forcePrompt = azureConfig.groupMap[groupName].forcePrompt;
       this.azure = !serverless && azureOptions;
+      if (serverless === true) {
+        this.options.defaultQuery = azureOptions.azureOpenAIApiVersion
+          ? { 'api-version': azureOptions.azureOpenAIApiVersion }
+          : undefined;
+        this.options.headers['api-key'] = this.apiKey;
+      }
     }
 
     const titleChatCompletion = async () => {
@@ -976,7 +982,7 @@ ${convo}
     let prompt;
 
     // TODO: remove the gpt fallback and make it specific to endpoint
-    const { OPENAI_SUMMARY_MODEL = 'gpt-3.5-turbo' } = process.env ?? {};
+    const { OPENAI_SUMMARY_MODEL = 'gpt-4o-mini' } = process.env ?? {};
     let model = this.options.summaryModel ?? OPENAI_SUMMARY_MODEL;
     if (model === Constants.CURRENT_MODEL) {
       model = this.modelOptions.model;
@@ -1169,6 +1175,10 @@ ${convo}
         opts.defaultHeaders = { ...opts.defaultHeaders, ...this.options.headers };
       }
 
+      if (this.options.defaultQuery) {
+        opts.defaultQuery = this.options.defaultQuery;
+      }
+
       if (this.options.proxy) {
         opts.httpAgent = new HttpsProxyAgent(this.options.proxy);
       }
@@ -1207,6 +1217,12 @@ ${convo}
         this.azure = !serverless && azureOptions;
         this.azureEndpoint =
           !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+        if (serverless === true) {
+          this.options.defaultQuery = azureOptions.azureOpenAIApiVersion
+            ? { 'api-version': azureOptions.azureOpenAIApiVersion }
+            : undefined;
+          this.options.headers['api-key'] = this.apiKey;
+        }
       }
 
       if (this.azure || this.options.azure) {
@@ -1308,6 +1324,11 @@ ${convo}
       /** @type {(value: void | PromiseLike<void>) => void} */
       let streamResolve;
 
+      if (this.isO1Model === true && this.azure && modelOptions.stream) {
+        delete modelOptions.stream;
+        delete modelOptions.stop;
+      }
+
       if (modelOptions.stream) {
         streamPromise = new Promise((resolve) => {
           streamResolve = resolve;

diff --git a/api/app/clients/PluginsClient.js b/api/app/clients/PluginsClient.js
@@ -105,7 +105,7 @@ class PluginsClient extends OpenAIClient {
       chatHistory: new ChatMessageHistory(pastMessages),
     });
 
-    this.tools = await loadTools({
+    const { loadedTools } = await loadTools({
       user,
       model,
       tools: this.options.tools,
@@ -119,12 +119,15 @@ class PluginsClient extends OpenAIClient {
         processFileURL,
         message,
       },
+      useSpecs: true,
     });
 
-    if (this.tools.length === 0) {
+    if (loadedTools.length === 0) {
       return;
     }
 
+    this.tools = loadedTools;
+
     logger.debug('[PluginsClient] Requested Tools', this.options.tools);
     logger.debug(
       '[PluginsClient] Loaded Tools',

diff --git a/api/app/clients/llm/createLLM.js b/api/app/clients/llm/createLLM.js
@@ -17,7 +17,7 @@ const { isEnabled } = require('~/server/utils');
  *
  * @example
  * const llm = createLLM({
- *   modelOptions: { modelName: 'gpt-3.5-turbo', temperature: 0.2 },
+ *   modelOptions: { modelName: 'gpt-4o-mini', temperature: 0.2 },
  *   configOptions: { basePath: 'https://example.api/path' },
  *   callbacks: { onMessage: handleMessage },
  *   openAIApiKey: 'your-api-key'