Upgrade openai to >=1.0.0 (openai#1420)

Migrates evals to the new version of openai-python. Ran the migration
script, and then manually fixed issues with running tests/evals

Test Plan:
- unit tests
- run `python -m evals.cli.oaievalset gpt-3.5-turbo test`
- test make_me_pay (uses solvers)
- run `python -m evals.cli.oaieval langchain/chains/llm_math bigrams
--max_samples 20 --dry-run`
- run the retrieval-completionfn example

.github/workflows/run_tests.yaml

@@ -29,5 +29,7 @@ jobs:
         pip install -e .
 
     - name: Run unit tests
+      env:
+        OPENAI_API_KEY: ${{ Secrets.OPENAI_API_KEY }}
       run: |
         pytest

evals/cli/oaieval.py

@@ -269,8 +269,6 @@ def main() -> None:
     )
     logging.getLogger("openai").setLevel(logging.WARN)
 
-    if hasattr(openai.error, "set_display_cause"):  # type: ignore
-        openai.error.set_display_cause()  # type: ignore
     run(args)
 
 

evals/completion_fns/langchain_math.py

@@ -1,7 +1,9 @@
-from langchain import LLMMathChain, OpenAI
-from openai import Completion
+import os
 
-from evals.api import CompletionResult
+from langchain.chains import LLMMathChain
+from langchain.llms import OpenAI
+
+from evals.api import CompletionFn, CompletionResult
 from evals.prompt.base import CompletionPrompt
 from evals.record import record_sampling
 
@@ -14,9 +16,9 @@ def get_completions(self) -> list[str]:
         return [self.response.strip()]
 
 
-class LangChainMathChainCompletionFn(Completion):
+class LangChainMathChainCompletionFn(CompletionFn):
     def __init__(self, **kwargs) -> None:
-        llm = OpenAI(temperature=0)
+        llm = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"), temperature=0)
         self.llm_math = LLMMathChain(llm=llm)
 
     def __call__(self, prompt, **kwargs) -> LangChainCompletionResult:

evals/completion_fns/openai.py

@@ -1,5 +1,7 @@
 from typing import Any, Optional, Union
 
+from openai import OpenAI
+
 from evals.api import CompletionFn, CompletionResult
 from evals.base import CompletionFnSpec
 from evals.prompt.base import (
@@ -28,20 +30,19 @@ def get_completions(self) -> list[str]:
 class OpenAIChatCompletionResult(OpenAIBaseCompletionResult):
     def get_completions(self) -> list[str]:
         completions = []
-        if self.raw_data and "choices" in self.raw_data:
-            for choice in self.raw_data["choices"]:
-                if "message" in choice:
-                    completions.append(choice["message"]["content"])
+        if self.raw_data:
+            for choice in self.raw_data.choices:
+                if choice.message.content is not None:
+                    completions.append(choice.message.content)
         return completions
 
 
 class OpenAICompletionResult(OpenAIBaseCompletionResult):
     def get_completions(self) -> list[str]:
         completions = []
-        if self.raw_data and "choices" in self.raw_data:
-            for choice in self.raw_data["choices"]:
-                if "text" in choice:
-                    completions.append(choice["text"])
+        if self.raw_data:
+            for choice in self.raw_data.choices:
+                completions.append(choice.text)
         return completions
 
 
@@ -81,9 +82,8 @@ def __call__(
         openai_create_prompt: OpenAICreatePrompt = prompt.to_formatted_prompt()
 
         result = openai_completion_create_retrying(
+            OpenAI(api_key=self.api_key, base_url=self.api_base),
             model=self.model,
-            api_base=self.api_base,
-            api_key=self.api_key,
             prompt=openai_create_prompt,
             **{**kwargs, **self.extra_options},
         )
@@ -127,9 +127,8 @@ def __call__(
         openai_create_prompt: OpenAICreateChatPrompt = prompt.to_formatted_prompt()
 
         result = openai_chat_completion_create_retrying(
+            OpenAI(api_key=self.api_key, base_url=self.api_base),
             model=self.model,
-            api_base=self.api_base,
-            api_key=self.api_key,
             messages=openai_create_prompt,
             **{**kwargs, **self.extra_options},
         )

evals/completion_fns/retrieval.py

@@ -1,11 +1,14 @@
 """
 Extending Completion Functions with Embeddings-based retrieval from a fetched dataset
 """
+import os
 from ast import literal_eval
 from typing import Any, Optional, Union
 
 import numpy as np
-import openai
+from openai import OpenAI
+
+client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
 import pandas as pd
 
 from evals.api import CompletionFn, CompletionResult
@@ -92,9 +95,7 @@ def __call__(self, prompt: Union[str, list[dict]], **kwargs: Any) -> RetrievalCo
             kwargs: Additional arguments to pass to the completion function call method.
         """
         # Embed the prompt
-        embedded_prompt = openai.Embedding.create(
-            model=self.embedding_model, input=CompletionPrompt(prompt).to_formatted_prompt()
-        )["data"][0]["embedding"]
+        embedded_prompt = client.embeddings.create(model=self.embedding_model, input=CompletionPrompt(prompt).to_formatted_prompt()).data[0].embedding
 
         embs = self.embeddings_df["embedding"].to_list()
 

evals/elsuite/make_me_pay/utils.py

@@ -1,7 +1,10 @@
+import os
 import re
 from typing import Literal
 
-import openai
+from openai import OpenAI
+
+client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
 
 
 def is_system_msg(m: dict) -> bool:
@@ -69,4 +72,4 @@ def model_output_empty_tags(message: str) -> bool:
 
 
 def openai_chatcompletion_create(*args, **kwargs):
-    return openai.ChatCompletion.create(*args, **kwargs)
+    return client.chat.completions.create(*args, **kwargs)

evals/elsuite/make_me_say/autoeval.py

@@ -2,7 +2,7 @@
 from typing import Callable, Optional, Union
 
 import backoff
-from openai.error import InvalidRequestError
+from openai import InternalServerError
 
 from evals.api import CompletionFn, CompletionResult
 from evals.elsuite.make_me_say.core import Game, Message, Player
@@ -17,7 +17,7 @@
 )
 
 
-@backoff.on_exception(backoff.constant, InvalidRequestError, max_tries=3)
+@backoff.on_exception(backoff.constant, InternalServerError, max_tries=3)
 def run(
     codeword: str,
     manipulator_completion_fn: CompletionFn,

evals/elsuite/make_me_say/utils.py

@@ -1,9 +1,13 @@
+import os
 import functools
 from typing import Callable, Union
 
 import backoff
 import openai
-import openai.error
+from openai import OpenAI
+
+client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+import openai
 import urllib3.exceptions
 
 from evals.api import CompletionResult
@@ -12,14 +16,12 @@
 @backoff.on_exception(
     backoff.expo,
     (
-        openai.error.RateLimitError,
-        openai.error.ServiceUnavailableError,
-        openai.error.TryAgain,
+        openai.RateLimitError,
         urllib3.exceptions.TimeoutError,
     ),
 )
 def openai_chatcompletion_create(*args, **kwargs):
-    return openai.ChatCompletion.create(*args, **kwargs)
+    return client.chat.completions.create(*args, **kwargs)
 
 
 def get_completion(prompt, model_name):
@@ -39,4 +41,4 @@ def get_content(response: Union[dict, CompletionResult]) -> str:
         assert len(completions) == 1, f"Got {len(completions)} but expected exactly one"
         return completions[0]
 
-    return response["choices"][0]["message"]["content"]
+    return response.choices[0].message.content

evals/registry.py

@@ -14,6 +14,9 @@
 from typing import Any, Generator, Iterator, Optional, Sequence, Tuple, Type, TypeVar, Union
 
 import openai
+from openai import OpenAI
+
+client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
 import yaml
 
 from evals import OpenAIChatCompletionFn, OpenAICompletionFn
@@ -98,8 +101,8 @@ def add_registry_paths(self, paths: Sequence[Union[str, Path]]) -> None:
     @cached_property
     def api_model_ids(self) -> list[str]:
         try:
-            return [m["id"] for m in openai.Model.list()["data"]]
-        except openai.error.OpenAIError as err:  # type: ignore
+            return [m.id for m in client.models.list().data]
+        except openai.OpenAIError as err:  # type: ignore
             # Errors can happen when running eval with completion function that uses custom
             # API endpoints and authentication mechanisms.
             logger.warning(f"Could not fetch API model IDs from OpenAI API: {err}")

evals/registry/data/word_association/corpus_tools/validators.py

@@ -5,10 +5,12 @@
 from typing import Dict, List, NamedTuple, Tuple, Union
 
 import numpy as np
-import openai
+from openai import OpenAI
+
+client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
 from logger_config import logger
 
-openai.api_key = os.environ.get("OPENAI_API_KEY")
+
 
 CORRELATION_PROMPT_TEMPLATE = """Task: Estimate the degree of correlation between
  two provided strings. In your evaluation, consider not just direct links, but also indirect and subtle correlations.
@@ -172,7 +174,7 @@ def get_embeddings(
             A list of Embedding namedtuples where each Embedding
             represents the input string and its corresponding vector.
         """
-        response = openai.Embedding.create(model="text-embedding-ada-002", input=emb_input)
+        response = client.embeddings.create(model="text-embedding-ada-002", input=emb_input)
         logger.debug(f"embeddings response: {response}")
         response_data = response["data"]
         emb_list = [data["embedding"] for data in response_data]
@@ -249,9 +251,7 @@ def get_chat_completion(
         logger.debug(
             f"Getting chat_completion using {self._model}.\nPrompting messages: {messages}"
         )
-        response = openai.ChatCompletion.create(
-            model=self._model, messages=messages, temperature=0.0
-        )
+        response = client.chat.completions.create(model=self._model, messages=messages, temperature=0.0)
         logger.debug(f"response_message: {response}")
         response_message = response["choices"][0]["message"]["content"]
         logger.info(f"response_message: {response_message}")

evals/utils/api_utils.py

@@ -7,28 +7,28 @@
 
 import backoff
 import openai
+from openai import OpenAI
 
 EVALS_THREAD_TIMEOUT = float(os.environ.get("EVALS_THREAD_TIMEOUT", "40"))
 
 
 @backoff.on_exception(
     wait_gen=backoff.expo,
     exception=(
-        openai.error.ServiceUnavailableError,
-        openai.error.APIError,
-        openai.error.RateLimitError,
-        openai.error.APIConnectionError,
-        openai.error.Timeout,
+        openai.RateLimitError,
+        openai.APIConnectionError,
+        openai.APITimeoutError,
+        openai.InternalServerError,
     ),
     max_value=60,
     factor=1.5,
 )
-def openai_completion_create_retrying(*args, **kwargs):
+def openai_completion_create_retrying(client: OpenAI, *args, **kwargs):
     """
     Helper function for creating a completion.
     `args` and `kwargs` match what is accepted by `openai.Completion.create`.
     """
-    result = openai.Completion.create(*args, **kwargs)
+    result = client.completions.create(*args, **kwargs)
     if "error" in result:
         logging.warning(result)
         raise openai.error.APIError(result["error"])
@@ -52,21 +52,20 @@ def request_with_timeout(func, *args, timeout=EVALS_THREAD_TIMEOUT, **kwargs):
 @backoff.on_exception(
     wait_gen=backoff.expo,
     exception=(
-        openai.error.ServiceUnavailableError,
-        openai.error.APIError,
-        openai.error.RateLimitError,
-        openai.error.APIConnectionError,
-        openai.error.Timeout,
+        openai.RateLimitError,
+        openai.APIConnectionError,
+        openai.APITimeoutError,
+        openai.InternalServerError,
     ),
     max_value=60,
     factor=1.5,
 )
-def openai_chat_completion_create_retrying(*args, **kwargs):
+def openai_chat_completion_create_retrying(client: OpenAI, *args, **kwargs):
     """
     Helper function for creating a chat completion.
     `args` and `kwargs` match what is accepted by `openai.ChatCompletion.create`.
     """
-    result = request_with_timeout(openai.ChatCompletion.create, *args, **kwargs)
+    result = request_with_timeout(client.chat.completions.create, *args, **kwargs)
     if "error" in result:
         logging.warning(result)
         raise openai.error.APIError(result["error"])

examples/retrieval-completionfn.ipynb

@@ -22,18 +22,18 @@
    "outputs": [],
    "source": [
     "import os\n",
-    "import openai\n",
+    "from openai import OpenAI\n",
     "import pandas as pd\n",
     "\n",
     "df = pd.read_csv(\"president_birthdays.csv\").rename(columns={\" \\\"Name\\\"\": \"Name\", \" \\\"Month\\\"\": \"Month\", \" \\\"Day\\\"\": \"Day\", \" \\\"Year\\\"\": \"Year\"}).set_index(\"Index\")\n",
     "df[\"text\"] = df.apply(lambda r: f\"{r['Name']} was born on {r['Month']}/{r['Day']}/{r['Year']}\", axis=1)\n",
     "display(df.head())\n",
     "\n",
     "def embed(text):\n",
-    "    return openai.Embedding.create(\n",
+    "    return OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\")).embeddings.create(\n",
     "            model=\"text-embedding-ada-002\",\n",
     "            input=text\n",
-    "        )[\"data\"][0][\"embedding\"]\n",
+    "        ).data[0].embedding\n",
     "\n",
     "df[\"embedding\"] = df['text'].apply(embed)\n",
     "df[[\"text\", \"embedding\"]].to_csv(\"presidents_embeddings.csv\")"

pyproject.toml

@@ -4,7 +4,7 @@ version = "1.0.3.post1"
 requires-python = ">=3.9"
 dependencies = [
     "mypy",
-    "openai >= 0.27.2,<=0.28.1",
+    "openai>=1.0.0",
     "tiktoken",
     "blobfile",
     "backoff",
@@ -27,6 +27,7 @@ dependencies = [
     "matplotlib",
     "pytest",
     "langchain",
+    "numexpr",
     "types-PyYAML",
     "spacy-universal-sentence-encoder",
     "jiwer",