Openai api & eval debug

.env.example

@@ -1,27 +1,14 @@
 # Global configs:
-USE_AZURE=True
 MAX_RETRY=10
 RETRY_WAIT_SECONDS=20
-DUMP_CHAT_CACHE=True
-USE_CHAT_CACHE=True
-DUMP_EMBEDDING_CACHE=True
-USE_EMBEDDING_CACHE=True
-LOG_LLM_CHAT_CONTENT=False
-CHAT_FREQUENCY_PENALTY=0.0
-CHAT_PRESENCE_PENALTY=0.0
-LOG_TRACE_PATH=log_traces
+
+# api key
+OPENAI_API_KEY=your_api_key
 
 # embedding model configs:
-EMBEDDING_OPENAI_API_KEY=your_api_key
-EMBEDDING_AZURE_API_BASE=your_api_base
-EMBEDDING_AZURE_API_VERSION=your_api_version
 EMBEDDING_MODEL=text-embedding-3-small
 
-
 # chat model configs:
-CHAT_OPENAI_API_KEY=your_api_key # 5c
-CHAT_AZURE_API_BASE=your_api_base
-CHAT_AZURE_API_VERSION=your_api_version
 CHAT_MODEL=your_model_version
 CHAT_MAX_TOKENS=3000
 CHAT_TEMPERATURE=0.7

README.md

@@ -60,8 +60,8 @@ TODO: use docker in quick start intead.
 
 ### ⚙️ Environment Configuration
 - Place the `.env` file in the same directory as the `.env.example` file.
-  - TOOD: please refer to ... for the detailed explanation of the `.env`
-  - TODO: simplify `.env.example`  only keep OpenAI or Azure Azure OpenAI
+  - The `.env.example` file contains the environment variables required for users using the OpenAI API
+  - please refer to [Configuration](docs/build/html/installation.html#azure-openai) for the detailed explanation of the `.env`
 - Export each variable in the `.env` file:
   ```sh
   export $(grep -v '^#' .env | xargs)

docs/installation.rst

@@ -12,20 +12,50 @@ For different scenarios
 Configuration
 =============
 
-Quick configuration
+To run the application, please create a `.env` file in the root directory of the project and add environment variables according to your requirements.
 
+The standard configuration options for the user using the OpenAI API are provided in the `.env.example` file.
 
+Here are some other configuration options that you can use:
 
+OpenAI API
+------------
+
+You can use different OpenAI API keys for embedding model and chat model.
+
+   .. code-block:: Properties
+
+      EMBEDDING_OPENAI_API_KEY=<replace_with_your_openai_api_key>
+      EMBEDDING_MODEL=text-embedding-3-small
+
+      CHAT_OPENAI_API_KEY=<replace_with_your_openai_api_key>
+      CHAT_MODEL=gpt-4-turbo
 
 Azure OpenAI
 ------------
 
+The following environment variables are standard configuration options for the user using the OpenAI API.
+
+   .. code-block:: Properties
+
+      USE_AZURE=True
+
+      OPENAI_API_KEY=<replace_with_your_openai_api_key>
+
+      EMBEDDING_MODEL=text-embedding-3-small
+      EMBEDDING_AZURE_API_BASE= # The base URL for the Azure OpenAI API.
+      EMBEDDING_AZURE_API_VERSION = # The version of the Azure OpenAI API.
+
+      CHAT_MODEL=gpt-4-turbo
+      CHAT_AZURE_API_VERSION = # The version of the Azure OpenAI API.
 
+Use Azure Token Provider
+------------------------
 
-USE_AZURE_TOKEN_PROVIDER
-~~~~~~~~~~~~~~~~~~~~~~~~
+If you are using the Azure token provider, you need to set the `USE_AZURE_TOKEN_PROVIDER` environment variable to `True`. then 
+use the environment variables provided in the `Azure Configuration section <installation.html#azure-openai>`_.
 
-### ☁️ Azure Configuration
+☁️ Azure Configuration
 - Install Azure CLI:
 
    ```sh

rdagent/app/quant_factor_benchmark/eval.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from pprint import pprint
 
-from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
+from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
 from rdagent.components.benchmark.conf import BenchmarkSettings
 from rdagent.components.benchmark.eval_method import FactorImplementEval
 from rdagent.core.scenario import Scenario
@@ -23,7 +23,7 @@
 
 # 3.declare the method to be tested and pass the arguments.
 
-scen: Scenario = import_class(PROP_SETTING.factor_scen)()
+scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
 generate_method = import_class(bs.bench_method_cls)(scen=scen)
 # 4.declare the eval method and pass the arguments.
 eval_method = FactorImplementEval(

rdagent/components/benchmark/eval_method.py

@@ -30,12 +30,6 @@
 ]
 
 
-EVAL_RES = Dict[
-    str,
-    List[Tuple[FactorEvaluator, Union[object, RunnerException]]],
-]
-
-
 class TestCase:
     def __init__(
         self,

rdagent/core/conf.py

@@ -34,6 +34,7 @@ class RDAgentSettings(BaseSettings):
     max_past_message_include: int = 10
 
     # Chat configs
+    openai_api_key: str = ""
     chat_openai_api_key: str = ""
     chat_azure_api_base: str = ""
     chat_azure_api_version: str = ""

rdagent/oai/llm_utils.py

@@ -298,17 +298,21 @@ def __init__(  # noqa: C901, PLR0912, PLR0915
             self.use_azure_token_provider = self.cfg.use_azure_token_provider
             self.managed_identity_client_id = self.cfg.managed_identity_client_id
 
-            self.chat_api_key = self.cfg.chat_openai_api_key if chat_api_key is None else chat_api_key
+            if self.cfg.openai_api_key:
+                self.chat_api_key = self.cfg.openai_api_key
+                self.embedding_api_key = self.cfg.openai_api_key
+            else:
+                self.chat_api_key = self.cfg.chat_openai_api_key if chat_api_key is None else chat_api_key
+                self.embedding_api_key = (
+                    self.cfg.embedding_openai_api_key if embedding_api_key is None else embedding_api_key
+                )
             self.chat_model = self.cfg.chat_model if chat_model is None else chat_model
             self.encoder = tiktoken.encoding_for_model(self.chat_model)
             self.chat_api_base = self.cfg.chat_azure_api_base if chat_api_base is None else chat_api_base
             self.chat_api_version = self.cfg.chat_azure_api_version if chat_api_version is None else chat_api_version
             self.chat_stream = self.cfg.chat_stream
             self.chat_seed = self.cfg.chat_seed
 
-            self.embedding_api_key = (
-                self.cfg.embedding_openai_api_key if embedding_api_key is None else embedding_api_key
-            )
             self.embedding_model = self.cfg.embedding_model if embedding_model is None else embedding_model
             self.embedding_api_base = (
                 self.cfg.embedding_azure_api_base if embedding_api_base is None else embedding_api_base
@@ -610,44 +614,25 @@ def _create_chat_completion_inner_function(  # noqa: C901, PLR0912, PLR0915
             if self.cfg.log_llm_chat_content:
                 logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
         else:
-            if self.use_azure:
-                if json_mode:
-                    if add_json_in_prompt:
-                        for message in messages[::-1]:
-                            message["content"] = message["content"] + "\nPlease respond in json format."
-                            if message["role"] == "system":
-                                break
-                    response = self.chat_client.chat.completions.create(
-                        model=self.chat_model,
-                        messages=messages,
-                        max_tokens=max_tokens,
-                        temperature=temperature,
-                        response_format={"type": "json_object"},
-                        stream=self.chat_stream,
-                        seed=self.chat_seed,
-                        frequency_penalty=frequency_penalty,
-                        presence_penalty=presence_penalty,
-                    )
-                else:
-                    response = self.chat_client.chat.completions.create(
-                        model=self.chat_model,
-                        messages=messages,
-                        max_tokens=max_tokens,
-                        temperature=temperature,
-                        stream=self.chat_stream,
-                        seed=self.chat_seed,
-                        frequency_penalty=frequency_penalty,
-                        presence_penalty=presence_penalty,
-                    )
-            else:
-                response = self.chat_client.chat.completions.create(
-                    model=self.chat_model,
-                    messages=messages,
-                    stream=self.chat_stream,
-                    seed=self.chat_seed,
-                    frequency_penalty=frequency_penalty,
-                    presence_penalty=presence_penalty,
-                )
+            kwargs = dict(
+                model=self.chat_model,
+                messages=messages,
+                max_tokens=max_tokens,
+                temperature=temperature,
+                stream=self.chat_stream,
+                seed=self.chat_seed,
+                frequency_penalty=frequency_penalty,
+                presence_penalty=presence_penalty,
+            )
+            if json_mode:
+                if add_json_in_prompt:
+                    for message in messages[::-1]:
+                        message["content"] = message["content"] + "\nPlease respond in json format."
+                        if message["role"] == "system":
+                            break
+                kwargs["response_format"] = {"type": "json_object"}
+            response = self.chat_client.chat.completions.create(**kwargs)
+
             if self.chat_stream:
                 resp = ""
                 # TODO: with logger.config(stream=self.chat_stream): and add a `stream_start` flag to add timestamp for first message.