From a830ecc1a25328b7118e56fe44b721e347f5e963 Mon Sep 17 00:00:00 2001 From: Jeremiah Lowin <153965+jlowin@users.noreply.github.com> Date: Tue, 16 Jan 2024 21:57:36 -0500 Subject: [PATCH 1/4] Azure support --- cookbook/azure/README.md | 25 ++++---- cookbook/azure/usage.py | 27 ++++++--- src/marvin/client/openai.py | 61 ++----------------- src/marvin/settings.py | 11 ++++ src/marvin/types.py | 8 +-- src/marvin/utilities/openai.py | 103 ++++++++++++++++++++++++++------- 6 files changed, 132 insertions(+), 103 deletions(-) diff --git a/cookbook/azure/README.md b/cookbook/azure/README.md index fe261ddb6..37d07ff1a 100644 --- a/cookbook/azure/README.md +++ b/cookbook/azure/README.md @@ -6,19 +6,22 @@ It is possible to use Azure OpenAI with _some_ of `marvin`'s functionality via: !!! Note Azure OpenAI often lags behind the latest version of OpenAI in terms of functionality, therefore some features may not work with Azure OpenAI. If you encounter problems, please check that the underlying functionality is supported by Azure OpenAI before reporting an issue. -## Settings -After setting up your Azure OpenAI account and deployment, it is recommended you save these settings in your `~/.marvin/.env` file. +## Configuring with environment variables +After setting up your Azure OpenAI account and deployment, set these environment variables in your environment or your `~/.marvin/.env` file: ```bash -ยป cat ~/.marvin/.env | rg AZURE -MARVIN_USE_AZURE_OPENAI=true +MARVIN_PROVIDER=azure_openai MARVIN_AZURE_OPENAI_API_KEY= -MARVIN_AZURE_OPENAI_ENDPOINT=https://.openai.azure.com/ -MARVIN_AZURE_OPENAI_API_VERSION=2023-12-01-preview # or whatever is the latest -MARVIN_AZURE_OPENAI_DEPLOYMENT_NAME=gpt-35-turbo-0613 # or whatever you named your deployment +MARVIN_AZURE_OPENAI_ENDPOINT="https://.openai.azure.com/" +MARVIN_AZURE_OPENAI_API_VERSION=2023-12-01-preview # or latest +MARVIN_CHAT_COMPLETION_MODEL= ``` -## Passing a `MarvinClient` set up with `AzureOpenAI` manually +Note that the chat completion model must be your Azure OpenAI deployment name. + +## Passing clients manually + +As an alternative to setting environment variables, you can pass the `AzureOpenAI` client to Marvin's components manually: ```python import marvin @@ -27,14 +30,14 @@ from marvin.client import MarvinClient from openai import AzureOpenAI azure_openai_client = AzureOpenAI( - api_key="my-api-key", - azure_endpoint="https://my-endpoint.openai.azure.com/", + api_key="your-api-key", + azure_endpoint="https://your-endpoint.openai.azure.com/", api_version="2023-12-01-preview", ) @marvin.fn( client=MarvinClient(client=azure_openai_client), - model_kwargs={"model": "gpt-35-turbo-0613"} + model_kwargs={"model": "your_deployment_name"} ) def list_fruits(n: int) -> list[str]: """generate a list of fruits""" diff --git a/cookbook/azure/usage.py b/cookbook/azure/usage.py index 98431d757..ad0aff4b2 100644 --- a/cookbook/azure/usage.py +++ b/cookbook/azure/usage.py @@ -1,14 +1,19 @@ -"""Usage example of Marvin with Azure OpenAI +""" +Usage example of Marvin with Azure OpenAI -If you'll be using Azure OpenAI exclusively, you can set the following environment variables in `~/.marvin/.env`: +If you'll be using Azure OpenAI exclusively, you can set the following env vars in your environment or `~/.marvin/.env`: ```bash -MARVIN_USE_AZURE_OPENAI=true -MARVIN_AZURE_OPENAI_API_KEY=... -MARVIN_AZURE_OPENAI_API_VERSION=... -MARVIN_AZURE_OPENAI_ENDPOINT=... -MARVIN_AZURE_OPENAI_DEPLOYMENT_NAME=... + +MARVIN_PROVIDER=azure_openai +MARVIN_AZURE_OPENAI_API_KEY= +MARVIN_AZURE_OPENAI_ENDPOINT="https://.openai.azure.com/" +MARVIN_AZURE_OPENAI_API_VERSION=2023-12-01-preview # or latest + +Note that you MUST set the LLM model name to be your Azure OpenAI deployment name, e.g. +MARVIN_CHAT_COMPLETION_MODEL= ``` """ + from enum import Enum import marvin @@ -34,8 +39,12 @@ def list_fruits(n: int = 3) -> list[str]: with temporary_settings( - use_azure_openai=True -): # or set MARVIN_USE_AZURE_OPENAI=true in `~/.marvin/.env` + provider="azure_openai", + azure_openai_api_key="...", + azure_openai_api_version="...", + azure_openai_endpoint="...", + chat_completion_model="", +): fruits = list_fruits() location = marvin.model(Location)("windy city") casted_location = marvin.cast("windy city", Location) diff --git a/src/marvin/client/openai.py b/src/marvin/client/openai.py index 1dcd65c3a..02b3d9150 100644 --- a/src/marvin/client/openai.py +++ b/src/marvin/client/openai.py @@ -1,4 +1,3 @@ -import inspect from functools import partial from pathlib import Path from typing import ( @@ -32,6 +31,7 @@ VisionRequest, ) from marvin.utilities.logging import get_logger +from marvin.utilities.openai import get_openai_client if TYPE_CHECKING: from openai.types import ImagesResponse @@ -143,65 +143,14 @@ async def should_fallback(e: NotFoundError, request: ChatRequest) -> bool: return False -def _get_default_client(client_type: str) -> Union[Client, AsyncClient]: - if getattr(settings, "use_azure_openai", False): - from openai import AsyncAzureOpenAI, AzureOpenAI - - client_class = AsyncAzureOpenAI if client_type == "async" else AzureOpenAI - - try: - return client_class( - api_key=settings.azure_openai_api_key, - api_version=settings.azure_openai_api_version, - azure_endpoint=settings.azure_openai_endpoint, - ) - except AttributeError: - raise ValueError( - inspect.cleandoc( - """ - To use Azure OpenAI, please set all of the following environment variables in `~/.marvin/.env`: - - ``` - MARVIN_USE_AZURE_OPENAI=true - MARVIN_AZURE_OPENAI_API_KEY=... - MARVIN_AZURE_OPENAI_API_VERSION=... - MARVIN_AZURE_OPENAI_ENDPOINT=... - MARVIN_AZURE_OPENAI_DEPLOYMENT_NAME=... - ``` - """ - ) - ) - - api_key = ( - settings.openai.api_key.get_secret_value() if settings.openai.api_key else None - ) - if not api_key: - raise ValueError( - inspect.cleandoc( - """ - OpenAI API key not found! Marvin will not work properly without it. - - You can either: - 1. Set the `MARVIN_OPENAI_API_KEY` or `OPENAI_API_KEY` environment variables - 2. Set `marvin.settings.openai.api_key` in your code (not recommended for production) - - If you do not have an OpenAI API key, you can create one at https://platform.openai.com/api-keys. - """ - ) - ) - if client_type not in ["sync", "async"]: - raise ValueError(f"Invalid client type {client_type!r}") - - client_class = Client if client_type == "sync" else AsyncClient - return client_class(api_key=api_key, organization=settings.openai.organization) - - class MarvinClient(pydantic.BaseModel): model_config = pydantic.ConfigDict( arbitrary_types_allowed=True, protected_namespaces=() ) - client: Client = pydantic.Field(default_factory=lambda: _get_default_client("sync")) + client: Client = pydantic.Field( + default_factory=lambda: get_openai_client(is_async=False) + ) @classmethod def wrap(cls, client: Client) -> "Client": @@ -287,7 +236,7 @@ class AsyncMarvinClient(pydantic.BaseModel): ) client: AsyncClient = pydantic.Field( - default_factory=lambda: _get_default_client("async") + default_factory=lambda: get_openai_client(is_async=True) ) @classmethod diff --git a/src/marvin/settings.py b/src/marvin/settings.py index 3079a9834..14b5df4a5 100644 --- a/src/marvin/settings.py +++ b/src/marvin/settings.py @@ -224,9 +224,20 @@ class Settings(MarvinSettings): protected_namespaces=(), ) + # providers + provider: Literal["openai", "azure_openai"] = Field( + default="openai", + description=( + 'The LLM provider to use. Supports "openai" and "azure_openai" at this' + " time." + ), + ) openai: OpenAISettings = Field(default_factory=OpenAISettings) + + # ai settings ai: AISettings = Field(default_factory=AISettings) + # log settings log_level: str = Field( default="INFO", description="The log level to use.", diff --git a/src/marvin/types.py b/src/marvin/types.py index 3aaaeec67..a7621ea0e 100644 --- a/src/marvin/types.py +++ b/src/marvin/types.py @@ -124,13 +124,7 @@ class ResponseModel(MarvinType): class ChatRequest(Prompt[T]): - model: str = Field( - default_factory=lambda: ( - settings.openai.chat.completions.model - if not getattr(settings, "use_azure_openai", False) - else settings.azure_openai_deployment_name - ) - ) + model: str = Field(default_factory=lambda: settings.openai.chat.completions.model) frequency_penalty: Optional[ Annotated[float, Field(strict=True, ge=-2.0, le=2.0)] ] = 0 diff --git a/src/marvin/utilities/openai.py b/src/marvin/utilities/openai.py index e7ae1520f..2d0404536 100644 --- a/src/marvin/utilities/openai.py +++ b/src/marvin/utilities/openai.py @@ -1,18 +1,24 @@ -"""Module for working with OpenAI.""" +"""Utilities for working with OpenAI.""" import asyncio +import inspect from functools import lru_cache -from typing import Optional +from typing import Any, Optional, Union -from openai import AsyncClient +import pydantic +from openai import AsyncAzureOpenAI, AsyncClient, AzureOpenAI, Client +import marvin -def get_openai_client() -> AsyncClient: + +def get_openai_client( + is_async: bool = True, +) -> Union[AsyncClient, Client, AzureOpenAI, AsyncAzureOpenAI]: """ - Retrieves an OpenAI client with the given api key and organization. + Retrieves an OpenAI client (sync or async) based on the current configuration. Returns: - The OpenAI client with the given api key and organization. + The OpenAI client Example: Retrieving an OpenAI client @@ -22,33 +28,90 @@ def get_openai_client() -> AsyncClient: client = get_client() ``` """ - from marvin import settings - api_key: Optional[str] = ( - settings.openai.api_key.get_secret_value() if settings.openai.api_key else None - ) - organization: Optional[str] = settings.openai.organization + kwargs = {} + + # --- Openai + if marvin.settings.provider == "openai": + client_class = AsyncClient if is_async else Client + + api_key = marvin.settings.openai.api_key + if isinstance(api_key, pydantic.SecretStr): + api_key = api_key.get_secret_value() + + if not api_key: + raise ValueError( + inspect.cleandoc( + """ + OpenAI API key not found! Marvin will not work properly without it. + + You can either: + 1. Set the `MARVIN_OPENAI_API_KEY` or `OPENAI_API_KEY` environment variables + 2. Set `marvin.settings.openai.api_key` in your code (not recommended for production) + + If you do not have an OpenAI API key, you can create one at https://platform.openai.com/api-keys. + """ + ) + ) + + kwargs.update(api_key=api_key, organization=marvin.settings.openai.organization) + + # --- Azure OpenAI + elif marvin.settings.provider == "azure_openai": + api_key = getattr(marvin.settings, "azure_openai_api_key", None) + api_version = getattr(marvin.settings, "azure_openai_api_version", None) + azure_endpoint = getattr(marvin.settings, "azure_openai_endpoint", None) + + if any(k is None for k in [api_key, api_version, azure_endpoint]): + raise ValueError( + inspect.cleandoc( + """ + Azure OpenAI configuration is missing. Marvin will not work properly without it. + + Please make sure to set the following environment variables: + - MARVIN_AZURE_OPENAI_API_KEY + - MARVIN_AZURE_OPENAI_API_VERSION + - MARVIN_AZURE_OPENAI_ENDPOINT + + In addition, you must set the LLM model name to your Azure OpenAI deployment name, e.g. + - MARVIN_CHAT_COMPLETION_MODEL = + """ + ) + ) + client_class = AsyncAzureOpenAI if is_async else AzureOpenAI + kwargs.update( + api_key=api_key, api_version=api_version, azure_endpoint=azure_endpoint + ) + + # --- N/A + else: + raise ValueError(f"Unknown provider {marvin.settings.provider}") + return _get_client_memoized( - api_key=api_key, organization=organization, loop=asyncio.get_event_loop() + cls=client_class, + loop=asyncio.get_event_loop(), + kwargs_items=tuple(kwargs.items()), ) @lru_cache def _get_client_memoized( - api_key: Optional[str], - organization: Optional[str], + cls: type, loop: Optional[asyncio.AbstractEventLoop] = None, -) -> AsyncClient: + kwargs_items: tuple[tuple[str, Any]] = None, +) -> Union[Client, AsyncClient]: """ This function is memoized to ensure that only one instance of the client is - created for a given api key / organization / loop tuple. + created for a given set of configuration parameters + + It can return either a sync or an async client. The `loop` is an important key to ensure that the client is not re-used across multiple event loops (which can happen when using the `run_sync` function). Attempting to re-use the client across multiple event loops can result in a `RuntimeError: Event loop is closed` error or infinite hangs. + + kwargs_items is a tuple of dict items to get around the fact that + memoization requires hashable arguments. """ - return AsyncClient( - api_key=api_key, - organization=organization, - ) + return cls(**dict(kwargs_items)) From 64f0207a27427a29e16d80112be16e82b448091e Mon Sep 17 00:00:00 2001 From: Jeremiah Lowin <153965+jlowin@users.noreply.github.com> Date: Tue, 16 Jan 2024 22:03:45 -0500 Subject: [PATCH 2/4] Update cookbook/azure/README.md Co-authored-by: nate nowack --- cookbook/azure/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbook/azure/README.md b/cookbook/azure/README.md index 37d07ff1a..3401f9f36 100644 --- a/cookbook/azure/README.md +++ b/cookbook/azure/README.md @@ -7,7 +7,7 @@ It is possible to use Azure OpenAI with _some_ of `marvin`'s functionality via: Azure OpenAI often lags behind the latest version of OpenAI in terms of functionality, therefore some features may not work with Azure OpenAI. If you encounter problems, please check that the underlying functionality is supported by Azure OpenAI before reporting an issue. ## Configuring with environment variables -After setting up your Azure OpenAI account and deployment, set these environment variables in your environment or your `~/.marvin/.env` file: +After setting up your Azure OpenAI account and deployment, set these environment variables in your environment, your `~/.marvin/.env` or `.env` file: ```bash MARVIN_PROVIDER=azure_openai From 6dc60601c5912981e7e274e245c58464ea95a43a Mon Sep 17 00:00:00 2001 From: Jeremiah Lowin <153965+jlowin@users.noreply.github.com> Date: Tue, 16 Jan 2024 22:07:26 -0500 Subject: [PATCH 3/4] Small tweaks --- cookbook/azure/README.md | 3 ++- cookbook/azure/usage.py | 2 +- docs/docs/configuration/settings.md | 16 +++++++++------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cookbook/azure/README.md b/cookbook/azure/README.md index 3401f9f36..52fb01ade 100644 --- a/cookbook/azure/README.md +++ b/cookbook/azure/README.md @@ -7,13 +7,14 @@ It is possible to use Azure OpenAI with _some_ of `marvin`'s functionality via: Azure OpenAI often lags behind the latest version of OpenAI in terms of functionality, therefore some features may not work with Azure OpenAI. If you encounter problems, please check that the underlying functionality is supported by Azure OpenAI before reporting an issue. ## Configuring with environment variables -After setting up your Azure OpenAI account and deployment, set these environment variables in your environment, your `~/.marvin/.env` or `.env` file: +After setting up your Azure OpenAI account and deployment, set these environment variables in your environment, `~/.marvin/.env`, or `.env` file: ```bash MARVIN_PROVIDER=azure_openai MARVIN_AZURE_OPENAI_API_KEY= MARVIN_AZURE_OPENAI_ENDPOINT="https://.openai.azure.com/" MARVIN_AZURE_OPENAI_API_VERSION=2023-12-01-preview # or latest + MARVIN_CHAT_COMPLETION_MODEL= ``` diff --git a/cookbook/azure/usage.py b/cookbook/azure/usage.py index ad0aff4b2..4a622e9bb 100644 --- a/cookbook/azure/usage.py +++ b/cookbook/azure/usage.py @@ -1,7 +1,7 @@ """ Usage example of Marvin with Azure OpenAI -If you'll be using Azure OpenAI exclusively, you can set the following env vars in your environment or `~/.marvin/.env`: +If you'll be using Azure OpenAI exclusively, you can set the following env vars in your environment, `~/.marvin/.env`, or `.env`: ```bash MARVIN_PROVIDER=azure_openai diff --git a/docs/docs/configuration/settings.md b/docs/docs/configuration/settings.md index fb0062236..b5515f066 100644 --- a/docs/docs/configuration/settings.md +++ b/docs/docs/configuration/settings.md @@ -30,13 +30,15 @@ A runtime settings object is accessible via `marvin.settings` and can be used to ## Settings for using Azure OpenAI models _Some_ of Marvin's functionality is supported by Azure OpenAI services. -If you're exclusively using Marvin with Azure OpenAI services, you can set the following environment variables to avoid having to pass `AzureOpenAI` client instances to Marvin's components. +After setting up your Azure OpenAI account and deployment, set these environment variables in your environment, `~/.marvin/.env`, or `.env` file: + ```bash -MARVIN_USE_AZURE_OPENAI=true -MARVIN_AZURE_OPENAI_API_KEY=... -MARVIN_AZURE_OPENAI_API_VERSION=... -MARVIN_AZURE_OPENAI_ENDPOINT=... -MARVIN_AZURE_OPENAI_DEPLOYMENT_NAME=... +MARVIN_PROVIDER=azure_openai +MARVIN_AZURE_OPENAI_API_KEY= +MARVIN_AZURE_OPENAI_ENDPOINT="https://.openai.azure.com/" +MARVIN_AZURE_OPENAI_API_VERSION=2023-12-01-preview # or latest + +MARVIN_CHAT_COMPLETION_MODEL= ``` -To selectively use Azure OpenAI services, you can pass an `AzureOpenAI` client to Marvin's components or use `temporary_settings` like this [example](https://github.com/PrefectHQ/marvin/blob/main/cookbook/azure/README.md). \ No newline at end of file +Note that the chat completion model must be your Azure OpenAI deployment name. \ No newline at end of file From 0d993d6f178af8ee1b1dd57a96cb13cea2a44a4b Mon Sep 17 00:00:00 2001 From: Jeremiah Lowin <153965+jlowin@users.noreply.github.com> Date: Tue, 16 Jan 2024 22:10:56 -0500 Subject: [PATCH 4/4] Fix secretstring parsing --- src/marvin/settings.py | 2 +- src/marvin/utilities/openai.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/marvin/settings.py b/src/marvin/settings.py index 14b5df4a5..8574de4fb 100644 --- a/src/marvin/settings.py +++ b/src/marvin/settings.py @@ -181,7 +181,7 @@ class OpenAISettings(MarvinSettings): audio: AudioSettings = Field(default_factory=AudioSettings) assistants: AssistantSettings = Field(default_factory=AssistantSettings) - @field_validator("api_key") + @field_validator("api_key", mode="before") def discover_api_key(cls, v): if v is None: # check global OpenAI API key diff --git a/src/marvin/utilities/openai.py b/src/marvin/utilities/openai.py index 2d0404536..8ea75a93f 100644 --- a/src/marvin/utilities/openai.py +++ b/src/marvin/utilities/openai.py @@ -5,7 +5,6 @@ from functools import lru_cache from typing import Any, Optional, Union -import pydantic from openai import AsyncAzureOpenAI, AsyncClient, AzureOpenAI, Client import marvin @@ -35,9 +34,11 @@ def get_openai_client( if marvin.settings.provider == "openai": client_class = AsyncClient if is_async else Client - api_key = marvin.settings.openai.api_key - if isinstance(api_key, pydantic.SecretStr): - api_key = api_key.get_secret_value() + api_key = ( + marvin.settings.openai.api_key.get_secret_value() + if marvin.settings.openai.api_key + else None + ) if not api_key: raise ValueError(