Merge pull request #1053 from expectedparrot/no_agent_info_if_not_set

No agent info if not set

edsl/agents/PromptConstructor.py

@@ -192,6 +192,10 @@ def agent_instructions_prompt(self) -> Prompt:
         >>> i.prompt_constructor.agent_instructions_prompt
         Prompt(text=\"""You are answering questions as if you were a human. Do not break character.\""")
         """
+        from edsl import Agent
+
+        if self.agent == Agent():  # if agent is empty, then return an empty prompt
+            return Prompt(text="")
         if not hasattr(self, "_agent_instructions_prompt"):
             applicable_prompts = prompt_lookup(
                 component_type="agent_instructions",

edsl/inference_services/OpenAIService.py

@@ -187,12 +187,15 @@ async def async_execute_model_call(
                 else:
                     content = user_prompt
                 client = self.async_client()
+                messages = [
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": content},
+                ]
+                if system_prompt == "" and self.omit_system_prompt_if_empty:
+                    messages = messages[1:]
                 params = {
                     "model": self.model,
-                    "messages": [
-                        {"role": "system", "content": system_prompt},
-                        {"role": "user", "content": content},
-                    ],
+                    "messages": messages,
                     "temperature": self.temperature,
                     "max_tokens": self.max_tokens,
                     "top_p": self.top_p,

edsl/language_models/LanguageModel.py

@@ -170,13 +170,16 @@ class LanguageModel(
     }  # TODO: Use the OpenAI Teir 1 rate limits
     _safety_factor = 0.8
 
-    def __init__(self, tpm=None, rpm=None, **kwargs):
+    def __init__(
+        self, tpm=None, rpm=None, omit_system_prompt_if_empty_string=True, **kwargs
+    ):
         """Initialize the LanguageModel."""
         self.model = getattr(self, "_model_", None)
         default_parameters = getattr(self, "_parameters_", None)
         parameters = self._overide_default_parameters(kwargs, default_parameters)
         self.parameters = parameters
         self.remote = False
+        self.omit_system_prompt_if_empty = omit_system_prompt_if_empty_string
 
         if rpm is not None:
             self._rpm = rpm

edsl/language_models/registry.py

@@ -38,12 +38,14 @@ class Model(metaclass=Meta):
     def __new__(cls, model_name=None, registry=None, *args, **kwargs):
         # Map index to the respective subclass
         if model_name is None:
-            model_name = cls.default_model
+            model_name = (
+                cls.default_model
+            )  # when model_name is None, use the default model, set in the config file
         from edsl.inference_services.registry import default
 
         registry = registry or default
 
-        if isinstance(model_name, int):
+        if isinstance(model_name, int):  # can refer to a model by index
             model_name = cls.available(name_only=True)[model_name]
 
         factory = registry.create_model_factory(model_name)

integration/test_all_questions_and_models.py

@@ -0,0 +1,117 @@
+from edsl import Model
+from edsl import Question, Survey
+from typing import List
+
+
+def test_question_types(include=None, exclude=None, permissive=True) -> Survey:
+    all = Question.available()
+    if include is not None:
+        all = [q for q in all if q in include]
+    if exclude is not None:
+        all = [q for q in all if q not in exclude]
+    questions = [Question.example(question_type) for question_type in all]
+    if permissive:
+        for q in questions:
+            q.permissive = True
+    return Survey(questions)
+
+
+def test_models(services=None) -> List[Model]:
+    if services is None:
+        services = Model.services()
+    models = []
+    for service in services:
+        models.extend(Model.available(service=service))
+    return [Model(model) for model, _, _ in models]
+
+
+types_to_test = [
+    "likert_five",
+    "linear_scale",
+    "rank",
+    "top_k",
+    "list",
+    "multiple_choice",
+    "numerical",
+    "yes_no",
+    "checkbox",
+    "extract",
+    "free_text",
+]
+# types_to_test = ["rank"]
+# types_to_test = ["likert_five"]
+services_to_test = Model.services()
+services_to_test.remove("ollama")
+services_to_test.remove("test")
+# services_to_test = ["azure"]  # , "groq", "bedrock"]
+# types_to_test = ["numerical"]
+for question_type in types_to_test:
+    print("Now testing question type: ", question_type)
+    for service in services_to_test:
+        print("Now testing service: ", service)
+        question_name = Question.example(question_type).question_name
+        # print("Now testing question type: ", question_type)
+        survey = test_question_types(include=[question_type], permissive=False)
+        models = test_models(services=[service])
+        results = survey.by(models).run(cache=True, skip_retry=True)
+
+        all_answers = (
+            results.select("answer.*")
+            .to_scenario_list()
+            .unpivot()
+            .select("value")
+            .to_list()
+        )
+        results.select(
+            "model.model",
+            "answer.*",
+            "generated_tokens.*",
+            f"{question_name}_cost",
+            f"{question_name}_one_usd_buys",
+        ).print()
+
+        frac_fail = sum([1 for answer in all_answers if answer is None]) / len(
+            all_answers
+        )
+        print(f"Fraction of failed answers: {frac_fail}")
+
+        if frac_fail > 0:
+            print("Failing models")
+            question_name = survey.questions[0].question_name
+            results.filter(f"answer.{question_name} is None").select(
+                "model.model"
+            ).print()
+
+
+# print("Now running against {} models".format(len(models)))
+# results = s.by(models).run(print_exceptions=True, skip_retry=True, cache=False)
+# info = results.push()
+# outcome = results.to_scenario_list()
+# info = outcome.push()
+# print(info)
+
+# combined_responses = None
+# for service in Model.services():
+#     if service in ["bedrock", "deep_infra"]:
+#         print(f"Testing {service}")
+#         models = [Model(model) for model, _, _ in Model.available(service=service)]
+#         # from edsl import QuestionNumerical as Q
+
+#         q = Q.example()
+#         q.min_selections = 1
+#         results = q.by(models).run()
+#         results.select("model", "answer.*").print()
+#         sl = (
+#             results.select("model", "answer.*")
+#             .to_scenario_list()
+#             .add_value("service", service)
+#         )
+#         if combined_responses is None:
+#             combined_responses = sl
+#         else:
+#             combined_responses += sl
+
+
+# combined_responses.print()
+# info = combined_responses.push()
+# print(info)