[FEAT] Groq LLM support (#865)

* Groq LLM support complete * update useGetProvidersModels for groq models * Add definiations update comments and error log reports add example envs --------- Co-authored-by: timothycarambat <[email protected]>
Mintplex-Labs · Mar 6, 2024 · 0634013 · 0634013
1 parent 4731ec8
commit 0634013
Show file tree

Hide file tree

Showing 13 changed files with 320 additions and 4 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -4,12 +4,15 @@
     "Astra",
     "Dockerized",
     "Embeddable",
+    "GROQ",
     "hljs",
+    "inferencing",
     "Langchain",
     "Milvus",
     "Mintplex",
     "Ollama",
     "openai",
+    "openrouter",
     "Qdrant",
     "vectordbs",
     "Weaviate",

diff --git a/docker/.env.example b/docker/.env.example
@@ -61,6 +61,10 @@ GID='1000'
 # HUGGING_FACE_LLM_API_KEY=hf_xxxxxx
 # HUGGING_FACE_LLM_TOKEN_LIMIT=8000
 
+# LLM_PROVIDER='groq'
+# GROQ_API_KEY=gsk_abcxyz
+# GROQ_MODEL_PREF=llama2-70b-4096
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

diff --git a/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx
@@ -0,0 +1,41 @@
+export default function GroqAiOptions({ settings }) {
+  return (
+    <div className="flex gap-x-4">
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Groq API Key
+        </label>
+        <input
+          type="password"
+          name="GroqApiKey"
+          className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="Groq API Key"
+          defaultValue={settings?.GroqApiKey ? "*".repeat(20) : ""}
+          required={true}
+          autoComplete="off"
+          spellCheck={false}
+        />
+      </div>
+
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Chat Model Selection
+        </label>
+        <select
+          name="GroqModelPref"
+          defaultValue={settings?.GroqModelPref || "llama2-70b-4096"}
+          required={true}
+          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          {["llama2-70b-4096", "mixtral-8x7b-32768"].map((model) => {
+            return (
+              <option key={model} value={model}>
+                {model}
+              </option>
+            );
+          })}
+        </select>
+      </div>
+    </div>
+  );
+}
diff --git a/frontend/src/hooks/useGetProvidersModels.js b/frontend/src/hooks/useGetProvidersModels.js
@@ -19,6 +19,7 @@ const PROVIDER_DEFAULT_MODELS = {
   localai: [],
   ollama: [],
   togetherai: [],
+  groq: ["llama2-70b-4096", "mixtral-8x7b-32768"],
   native: [],
 };
 

diff --git a/frontend/src/media/llmprovider/groq.png b/frontend/src/media/llmprovider/groq.png
diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
@@ -16,6 +16,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg";
 import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
 import PerplexityLogo from "@/media/llmprovider/perplexity.png";
 import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
+import GroqLogo from "@/media/llmprovider/groq.png";
 import PreLoader from "@/components/Preloader";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
 import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
@@ -28,11 +29,12 @@ import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions";
 import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions";
 import MistralOptions from "@/components/LLMSelection/MistralOptions";
 import HuggingFaceOptions from "@/components/LLMSelection/HuggingFaceOptions";
+import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions";
+import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
+import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import { MagnifyingGlass } from "@phosphor-icons/react";
-import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions";
-import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
 
 export default function GeneralLLMPreference() {
   const [saving, setSaving] = useState(false);
@@ -173,6 +175,14 @@ export default function GeneralLLMPreference() {
       options: <OpenRouterOptions settings={settings} />,
       description: "A unified interface for LLMs.",
     },
+    {
+      name: "Groq",
+      value: "groq",
+      logo: GroqLogo,
+      options: <GroqAiOptions settings={settings} />,
+      description:
+        "The fastest LLM inferencing available for real-time AI applications.",
+    },
     {
       name: "Native",
       value: "native",

diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
@@ -13,6 +13,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg";
 import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
 import PerplexityLogo from "@/media/llmprovider/perplexity.png";
 import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
+import GroqLogo from "@/media/llmprovider/groq.png";
 import ZillizLogo from "@/media/vectordbs/zilliz.png";
 import AstraDBLogo from "@/media/vectordbs/astraDB.png";
 import ChromaLogo from "@/media/vectordbs/chroma.png";
@@ -127,6 +128,14 @@ const LLM_SELECTION_PRIVACY = {
     ],
     logo: OpenRouterLogo,
   },
+  groq: {
+    name: "Groq",
+    description: [
+      "Your chats will not be used for training",
+      "Your prompts and document text used in response creation are visible to Groq",
+    ],
+    logo: GroqLogo,
+  },
 };
 
 const VECTOR_DB_PRIVACY = {

diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
@@ -13,6 +13,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg";
 import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
 import PerplexityLogo from "@/media/llmprovider/perplexity.png";
 import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
+import GroqLogo from "@/media/llmprovider/groq.png";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
 import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
 import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions";
@@ -25,12 +26,13 @@ import MistralOptions from "@/components/LLMSelection/MistralOptions";
 import HuggingFaceOptions from "@/components/LLMSelection/HuggingFaceOptions";
 import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions";
 import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions";
+import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
+import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions";
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import System from "@/models/system";
 import paths from "@/utils/paths";
 import showToast from "@/utils/toast";
 import { useNavigate } from "react-router-dom";
-import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
 
 const TITLE = "LLM Preference";
 const DESCRIPTION =
@@ -147,6 +149,14 @@ export default function LLMPreference({
       options: <OpenRouterOptions settings={settings} />,
       description: "A unified interface for LLMs.",
     },
+    {
+      name: "Groq",
+      value: "groq",
+      logo: GroqLogo,
+      options: <GroqAiOptions settings={settings} />,
+      description:
+        "The fastest LLM inferencing available for real-time AI applications.",
+    },
     {
       name: "Native",
       value: "native",

diff --git a/server/.env.example b/server/.env.example
@@ -58,6 +58,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
 # HUGGING_FACE_LLM_API_KEY=hf_xxxxxx
 # HUGGING_FACE_LLM_TOKEN_LIMIT=8000
 
+# LLM_PROVIDER='groq'
+# GROQ_API_KEY=gsk_abcxyz
+# GROQ_MODEL_PREF=llama2-70b-4096
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
@@ -219,12 +219,25 @@ const SystemSettings = {
             AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
           }
         : {}),
+
+      ...(llmProvider === "groq"
+        ? {
+            GroqApiKey: !!process.env.GROQ_API_KEY,
+            GroqModelPref: process.env.GROQ_MODEL_PREF,
+
+            // For embedding credentials when groq is selected.
+            OpenAiKey: !!process.env.OPEN_AI_KEY,
+            AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT,
+            AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY,
+            AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
+          }
+        : {}),
       ...(llmProvider === "native"
         ? {
             NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF,
             NativeLLMTokenLimit: process.env.NATIVE_LLM_MODEL_TOKEN_LIMIT,
 
-            // For embedding credentials when ollama is selected.
+            // For embedding credentials when native is selected.
             OpenAiKey: !!process.env.OPEN_AI_KEY,
             AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT,
             AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY,