feat: add openrouter support

README.md

@@ -90,6 +90,9 @@ SEARCH_PROVIDER=searxng
 # Cloud Models
 OPENAI_API_KEY=...
 GROQ_API_KEY=...
+# if you want to use openrouter, please provide the following variables
+OPENROUTER_API_KEY=...
+OPENROUTER_API_BASE=...
 ```
 
 ### 3. Run Containers

docker-compose.dev.yaml

@@ -11,6 +11,8 @@ services:
       - TAVILY_API_KEY=${TAVILY_API_KEY}
       - OPENAI_API_KEY=${OPENAI_API_KEY}
       - GROQ_API_KEY=${GROQ_API_KEY}
+      - OPENROUTER_API_KEY=${OPENROUTER_API_KEY}
+      - OPENROUTER_API_BASE=${OPENROUTER_API_BASE}
       - ENABLE_LOCAL_MODELS=${ENABLE_LOCAL_MODELS:-True}
       - SEARCH_PROVIDER=${SEARCH_PROVIDER:-tavily}
       - SEARXNG_BASE_URL=${SEARXNG_BASE_URL:-http://host.docker.internal:8080}

pyproject.toml

@@ -23,6 +23,7 @@ uvicorn = {extras = ["standard"], version = "^0.29.0"}
 logfire = {extras = ["fastapi"], version = "^0.32.0"}
 llama-index-llms-azure-openai = "^0.1.8"
 llama-index-llms-openai = "^0.1.19"
+llama-index-llms-openrouter = "^0.1.4"
 groq = "^0.5.0"
 slowapi = "^0.1.9"
 redis = "^5.0.4"

src/backend/Dockerfile.dev

@@ -16,10 +16,10 @@ WORKDIR /workspace
 # Copy dependency files to avoid cache invalidations
 COPY pyproject.toml poetry.lock ./
 
-COPY src/backend/ src/backend/
-
 RUN pip install poetry && poetry install
 
+COPY src/backend/ src/backend/
+
 EXPOSE 8000
 
 CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]

src/backend/chat.py

@@ -7,6 +7,7 @@
 from llama_index.llms.groq import Groq
 from llama_index.llms.ollama import Ollama
 from llama_index.llms.openai import OpenAI
+from llama_index.llms.openrouter import OpenRouter
 
 from backend.constants import ChatModel, model_mappings
 from backend.prompts import CHAT_PROMPT, HISTORY_QUERY_REPHRASE
@@ -48,6 +49,8 @@ def rephrase_query_with_history(question: str, history: List[Message], llm: LLM)
 def get_llm(model: ChatModel) -> LLM:
     if model in [ChatModel.GPT_3_5_TURBO, ChatModel.GPT_4o]:
         return OpenAI(model=model_mappings[model])
+    elif model in [ChatModel.OPENROUTER_GPT_3_5_TURBO, ChatModel.OPENROUTER_GPT_4o]:
+        return OpenRouter(model=model_mappings[model])
     elif model in [
         ChatModel.LOCAL_GEMMA,
         ChatModel.LOCAL_LLAMA_3,

src/backend/constants.py

@@ -1,7 +1,9 @@
 from enum import Enum
 
 GPT4_MODEL = "gpt-4o"
+OPENROUTER_GPT4_MODEL = "openai/gpt-4o"
 GPT3_MODEL = "gpt-3.5-turbo"
+OPENROUTER_GPT3_MODEL = "openai/gpt-3.5-turbo"
 LLAMA_8B_MODEL = "llama3-8b-8192"
 LLAMA_70B_MODEL = "llama3-70b-8192"
 
@@ -14,7 +16,9 @@
 class ChatModel(str, Enum):
     LLAMA_3_70B = "llama-3-70b"
     GPT_4o = "gpt-4o"
+    OPENROUTER_GPT_4o = "openai/gpt-4o"
     GPT_3_5_TURBO = "gpt-3.5-turbo"
+    OPENROUTER_GPT_3_5_TURBO = "openai/gpt-3.5-turbo"
 
     # Local models
     LOCAL_LLAMA_3 = "llama3"
@@ -25,7 +29,9 @@ class ChatModel(str, Enum):
 
 model_mappings: dict[ChatModel, str] = {
     ChatModel.GPT_3_5_TURBO: GPT3_MODEL,
+    ChatModel.OPENROUTER_GPT_3_5_TURBO: OPENROUTER_GPT3_MODEL,
     ChatModel.GPT_4o: GPT4_MODEL,
+    ChatModel.OPENROUTER_GPT_4o: OPENROUTER_GPT4_MODEL,
     ChatModel.LLAMA_3_70B: LLAMA_70B_MODEL,
     ChatModel.LOCAL_LLAMA_3: LOCAL_LLAMA3_MODEL,
     ChatModel.LOCAL_GEMMA: LOCAL_GEMMA_MODEL,

src/backend/related_queries.py

@@ -13,6 +13,15 @@
 
 
 OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
+# Extract API key from environment
+openrouter_api_key = os.environ.get("OPENROUTER_API_KEY")
+assert openrouter_api_key, "OPENROUTER_API_KEY is not set in environment variables"
+
+# Base URL for OpenAI client
+openrouter_api_base = os.environ.get("OPENROUTER_API_BASE")
+assert openrouter_api_base, "OPENROUTER_BASE_URL is not set in environment variables"
+
+
 
 
 def instructor_client(model: ChatModel) -> instructor.AsyncInstructor:
@@ -21,6 +30,17 @@ def instructor_client(model: ChatModel) -> instructor.AsyncInstructor:
         ChatModel.GPT_4o,
     ]:
         return instructor.from_openai(openai.AsyncOpenAI())
+    if model in [
+        ChatModel.OPENROUTER_GPT_3_5_TURBO,
+        ChatModel.OPENROUTER_GPT_4o,
+    ]:
+        return instructor.from_openai(
+            openai.AsyncOpenAI(
+                base_url=openrouter_api_base,
+                api_key=openrouter_api_key,
+            ),
+            mode=instructor.Mode.JSON,
+        )
     elif model in [
         ChatModel.LOCAL_GEMMA,
         ChatModel.LOCAL_LLAMA_3,

src/backend/validators.py

@@ -15,7 +15,10 @@ def validate_model(model: ChatModel):
                 raise ValueError(
                     "GPT4-o has been disabled. Please try a different model or self-host the app by following the instructions here: https://github.com/rashadphz/farfalle"
                 )
-
+    elif model in {ChatModel.OPENROUTER_GPT_3_5_TURBO, ChatModel.OPENROUTER_GPT_4o}:
+        OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
+        if not OPENROUTER_API_KEY:
+            raise ValueError("OPENROUTER_API_KEY environment variable not found")
     elif model == ChatModel.LLAMA_3_70B:
         GROQ_API_KEY = os.getenv("GROQ_API_KEY")
         if not GROQ_API_KEY:

src/frontend/Dockerfile.dev

@@ -1,7 +1,5 @@
 FROM node:18-alpine
 
-RUN apk add --no-cache libc6-compat
-
 WORKDIR /app/src/frontend
 
 COPY . /app

src/frontend/generated/types.gen.ts

@@ -6,13 +6,15 @@ export type BeginStream = {
 };
 
 export enum ChatModel {
-  LLAMA_3_70B = "llama-3-70b",
-  GPT_4O = "gpt-4o",
-  GPT_3_5_TURBO = "gpt-3.5-turbo",
-  LLAMA3 = "llama3",
-  GEMMA = "gemma",
-  MISTRAL = "mistral",
-  LOCAL_PHI3_14B = "phi3:14b",
+    LLAMA_3_70B = 'llama-3-70b',
+    GPT_4O = 'gpt-4o',
+    OPENROUTER_GPT_4O = 'openai/gpt-4o',
+    GPT_3_5_TURBO = 'gpt-3.5-turbo',
+    OPENROUTER_GPT_3_5_TURBO = 'openai/gpt-3.5-turbo',
+    LLAMA3 = 'llama3',
+    GEMMA = 'gemma',
+    MISTRAL = 'mistral',
+    LOCAL_PHI3_14B = "phi3:14b",
 }
 
 export type ChatRequest = {