From 6645db6131f20d575da74b3730154d196d20ba0a Mon Sep 17 00:00:00 2001 From: robin Date: Fri, 20 Sep 2024 09:27:23 -0400 Subject: [PATCH] docs page for prompts needs to be rewritten for version update - not done --- docs/prompts.rst | 343 +++++++++++++++++++---------------------------- 1 file changed, 137 insertions(+), 206 deletions(-) diff --git a/docs/prompts.rst b/docs/prompts.rst index bc1df6ef..45e74829 100644 --- a/docs/prompts.rst +++ b/docs/prompts.rst @@ -5,242 +5,173 @@ Prompts Overview -------- + Prompts are texts that are sent to a language model in order to guide it on how to generate responses to questions. -They consist of `agent instructions` and `question instructions`, and can include questions, instructions or any other text to be displayed to the language model. +Agent instructions are contained in a `system_prompt` and question instructions are contained in a `user_prompt`. +These texts can include questions, instructions or any other text to be displayed to the language model. Typically, prompts are created using the `Prompt` class, a subclass of the `PromptBase` class which is an abstract class that defines the basic structure of a prompt. Default prompts are provided in the `edsl.prompts.library` module. These prompts can be used as is or customized to suit specific requirements by creating new classes that inherit from the `Prompt` class. - -Default prompts -^^^^^^^^^^^^^^^ -The `edsl.prompts.library` module contains default prompts for agent instructions and question instructions (shown below). -If custom prompts are not specified, the default prompts used to generate results can be readily inspected by selecting the **prompt** columns in the results. -For example, we can inspect the prompts for the sample results generated in the `edsl.results` section: - -.. code-block:: python - - results.select("prompt.*").print(pretty_labels={ - "prompt.tomorrow_user_prompt": "Tomorrow: question instruction", - "prompt.tomorrow_system_prompt": "Tomorrow: agent instruction", - "prompt.yesterday_user_prompt": "Yesterday: question instruction", - "prompt.yesterday_system_prompt": "Yesterday: agent instruction" - }) - -.. code-block:: text - - ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ - ┃ Yesterday: question ┃ Tomorrow: agent ┃ Yesterday: agent ┃ Tomorrow: question ┃ - ┃ instruction ┃ instruction ┃ instruction ┃ instruction ┃ - ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ - │ {'text': 'You are being │ {'text': "You are │ {'text': "You are │ {'text': 'You are being │ - │ asked the following │ answering questions as if │ answering questions as if │ asked the following │ - │ question: How did you feel │ you were a human. Do not │ you were a human. Do not │ question: How do you │ - │ yesterday morning?\nThe │ break character. You are │ break character. You are │ expect to feel tomorrow │ - │ options are\n\n0: │ an agent with the │ an agent with the │ morning?\nReturn a valid │ - │ Good\n\n1: OK\n\n2: │ following │ following │ JSON formatted like │ - │ Terrible\n\nReturn a valid │ persona:\n{'status': │ persona:\n{'status': │ this:\n{"answer": ""}', 'class_name': │ - │ of the option:\n{"answer": │ │ │ 'FreeText'} │ - │ , │ │ │ │ - │ "comment": ""}\nOnly │ │ │ │ - │ 1 option may be │ │ │ │ - │ selected.', 'class_name': │ │ │ │ - │ 'MultipleChoiceTurbo'} │ │ │ │ - ├────────────────────────────┼───────────────────────────┼────────────────────────────┼───────────────────────────┤ - - ... +Note: If an `Agent` is not used with a survey the `system_prompt` base text is not sent to the model. Showing prompts ^^^^^^^^^^^^^^^ -Before you run a survey, EDSL creates a `Jobs` object. You can see the prompts it will use by calling `prompts()` on the `Jobs` object. + +Before a survey is run, EDSL creates a `Jobs` object. +You can see the prompts it will use by calling `prompts()` on it. For example: .. code-block:: python - from edsl import Model, Survey - j = Survey.example().by(Model()) - j.prompts().print() - -This will display the prompts that will be used in the survey: + from edsl import Survey, Agent, Model -.. code-block:: text + survey = Survey.example() + agent = Agent(traits = {"persona": "School teacher"}) + model = Model() # default model - ┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ - ┃ interview_index ┃ question_index ┃ user_prompt ┃ scenario_index ┃ system_prompt ┃ - ┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ - │ 0 │ q0 │ You are being asked the │ Scenario Attributes │ You are answering │ - │ │ │ following question: Do │ ┏━━━━━━━━━━━━┳━━━━━━━┓ │ questions as if you were │ - │ │ │ you like school? │ ┃ Attribute ┃ Value ┃ │ a human. Do not break │ - │ │ │ The options are │ ┡━━━━━━━━━━━━╇━━━━━━━┩ │ character. You are an │ - │ │ │ │ │ data │ {} │ │ agent with the following │ - │ │ │ 0: yes │ │ name │ None │ │ persona: │ - │ │ │ │ │ _has_image │ False │ │ {} │ - │ │ │ 1: no │ └────────────┴───────┘ │ │ - │ │ │ │ │ │ - │ │ │ Return a valid JSON │ │ │ - │ │ │ formatted like this, │ │ │ - │ │ │ selecting only the │ │ │ - │ │ │ number of the option: │ │ │ - │ │ │ {"answer": , "comment": │ │ │ - │ │ │ ""} │ │ │ - │ │ │ Only 1 option may be │ │ │ - │ │ │ selected. │ │ │ - ├─────────────────┼────────────────┼──────────────────────────┼────────────────────────┼──────────────────────────┤ - │ 0 │ q1 │ You are being asked the │ Scenario Attributes │ You are answering │ - │ │ │ following question: Why │ ┏━━━━━━━━━━━━┳━━━━━━━┓ │ questions as if you were │ - │ │ │ not? │ ┃ Attribute ┃ Value ┃ │ a human. Do not break │ - │ │ │ The options are │ ┡━━━━━━━━━━━━╇━━━━━━━┩ │ character. You are an │ - │ │ │ │ │ data │ {} │ │ agent with the following │ - │ │ │ 0: killer bees in │ │ name │ None │ │ persona: │ - │ │ │ cafeteria │ │ _has_image │ False │ │ {} │ - │ │ │ │ └────────────┴───────┘ │ │ - │ │ │ 1: other │ │ │ - │ │ │ │ │ │ - │ │ │ Return a valid JSON │ │ │ - │ │ │ formatted like this, │ │ │ - │ │ │ selecting only the │ │ │ - │ │ │ number of the option: │ │ │ - │ │ │ {"answer": , "comment": │ │ │ - │ │ │ ""} │ │ │ - │ │ │ Only 1 option may be │ │ │ - │ │ │ selected. │ │ │ - ├─────────────────┼────────────────┼──────────────────────────┼────────────────────────┼──────────────────────────┤ - │ 0 │ q2 │ You are being asked the │ Scenario Attributes │ You are answering │ - │ │ │ following question: Why? │ ┏━━━━━━━━━━━━┳━━━━━━━┓ │ questions as if you were │ - │ │ │ The options are │ ┃ Attribute ┃ Value ┃ │ a human. Do not break │ - │ │ │ │ ┡━━━━━━━━━━━━╇━━━━━━━┩ │ character. You are an │ - │ │ │ 0: **lack*** of killer │ │ data │ {} │ │ agent with the following │ - │ │ │ bees in cafeteria │ │ name │ None │ │ persona: │ - │ │ │ │ │ _has_image │ False │ │ {} │ - │ │ │ 1: other │ └────────────┴───────┘ │ │ - │ │ │ │ │ │ - │ │ │ Return a valid JSON │ │ │ - │ │ │ formatted like this, │ │ │ - │ │ │ selecting only the │ │ │ - │ │ │ number of the option: │ │ │ - │ │ │ {"answer": , "comment": │ │ │ - │ │ │ ""} │ │ │ - │ │ │ Only 1 option may be │ │ │ - │ │ │ selected. │ │ │ - └─────────────────┴────────────────┴──────────────────────────┴────────────────────────┴──────────────────────────┘ - - -Agent instructions -^^^^^^^^^^^^^^^^^^ -The `AgentInstruction` class provides guidance to a language model on how an agent should be represented. -As shown in the example above, the default agent instructions are: + job = survey.by(agent).by(model) # Creating a job for the example survey using the agent and the default model -.. code-block:: python + job.prompts().print(format="rich") - class AgentInstruction(PromptBase): - \"\"\"Agent instructions for a human agent.\"\"\" - model = LanguageModelType.GPT_3_5_Turbo.value - component_type = ComponentTypes.AGENT_INSTRUCTIONS - default_instructions = textwrap.dedent( - \"\"\"\ - You are playing the role of a human answering survey questions. - Do not break character. - \"\"\" - ) +This will display the prompts that will be used when the survey is run: +.. code-block:: text -Question instructions -^^^^^^^^^^^^^^^^^^^^^ -The `QuestionInstruction` class provides guidance to a language model on how a question should be answered. -As shown in the example above, the following question instructions are: + ┏━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ + ┃ interview_index ┃ question_index ┃ user_prompt ┃ scenario_index ┃ system_prompt ┃ + ┡━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ + │ 0 │ q0 │ │ Scenario Attributes │ You are answering │ + │ │ │ Do you like school? │ ┏━━━━━━━━━━━┳━━━━━━━┓ │ questions as if you were │ + │ │ │ │ ┃ Attribute ┃ Value ┃ │ a human. Do not break │ + │ │ │ │ ┡━━━━━━━━━━━╇━━━━━━━┩ │ character. You are an │ + │ │ │ yes │ │ data │ {} │ │ agent with the following │ + │ │ │ │ │ name │ None │ │ persona: │ + │ │ │ no │ └───────────┴───────┘ │ {'persona': 'School │ + │ │ │ │ │ teacher'} │ + │ │ │ │ │ │ + │ │ │ Only 1 option may be │ │ │ + │ │ │ selected. │ │ │ + │ │ │ │ │ │ + │ │ │ Respond only with a │ │ │ + │ │ │ string corresponding to │ │ │ + │ │ │ one of the options. │ │ │ + │ │ │ │ │ │ + │ │ │ │ │ │ + │ │ │ After the answer, you │ │ │ + │ │ │ can put a comment │ │ │ + │ │ │ explaining why you chose │ │ │ + │ │ │ that option on the next │ │ │ + │ │ │ line. │ │ │ + ├─────────────────┼────────────────┼──────────────────────────┼───────────────────────┼───────────────────────────┤ + │ 0 │ q1 │ │ Scenario Attributes │ You are answering │ + │ │ │ Why not? │ ┏━━━━━━━━━━━┳━━━━━━━┓ │ questions as if you were │ + │ │ │ │ ┃ Attribute ┃ Value ┃ │ a human. Do not break │ + │ │ │ │ ┡━━━━━━━━━━━╇━━━━━━━┩ │ character. You are an │ + │ │ │ killer bees in cafeteria │ │ data │ {} │ │ agent with the following │ + │ │ │ │ │ name │ None │ │ persona: │ + │ │ │ other │ └───────────┴───────┘ │ {'persona': 'School │ + │ │ │ │ │ teacher'} │ + │ │ │ │ │ │ + │ │ │ Only 1 option may be │ │ │ + │ │ │ selected. │ │ │ + │ │ │ │ │ │ + │ │ │ Respond only with a │ │ │ + │ │ │ string corresponding to │ │ │ + │ │ │ one of the options. │ │ │ + │ │ │ │ │ │ + │ │ │ │ │ │ + │ │ │ After the answer, you │ │ │ + │ │ │ can put a comment │ │ │ + │ │ │ explaining why you chose │ │ │ + │ │ │ that option on the next │ │ │ + │ │ │ line. │ │ │ + ├─────────────────┼────────────────┼──────────────────────────┼───────────────────────┼───────────────────────────┤ + │ 0 │ q2 │ │ Scenario Attributes │ You are answering │ + │ │ │ Why? │ ┏━━━━━━━━━━━┳━━━━━━━┓ │ questions as if you were │ + │ │ │ │ ┃ Attribute ┃ Value ┃ │ a human. Do not break │ + │ │ │ │ ┡━━━━━━━━━━━╇━━━━━━━┩ │ character. You are an │ + │ │ │ **lack*** of killer bees │ │ data │ {} │ │ agent with the following │ + │ │ │ in cafeteria │ │ name │ None │ │ persona: │ + │ │ │ │ └───────────┴───────┘ │ {'persona': 'School │ + │ │ │ other │ │ teacher'} │ + │ │ │ │ │ │ + │ │ │ │ │ │ + │ │ │ Only 1 option may be │ │ │ + │ │ │ selected. │ │ │ + │ │ │ │ │ │ + │ │ │ Respond only with a │ │ │ + │ │ │ string corresponding to │ │ │ + │ │ │ one of the options. │ │ │ + │ │ │ │ │ │ + │ │ │ │ │ │ + │ │ │ After the answer, you │ │ │ + │ │ │ can put a comment │ │ │ + │ │ │ explaining why you chose │ │ │ + │ │ │ that option on the next │ │ │ + │ │ │ line. │ │ │ + └─────────────────┴────────────────┴──────────────────────────┴───────────────────────┴───────────────────────────┘ + + +After we run the survey, we can verify the prompts that were used by inspecting the `prompt.*` fields of the results: .. code-block:: python - class QuestionInstruction(PromptBase): - \"\"\"Question instructions for a multiple choice question.\"\"\" - - model = LanguageModelType.GPT_3_5_Turbo.value - component_type = ComponentTypes.QUESTION_INSTRUCTIONS - default_instructions = textwrap.dedent( - \"\"\"\ - You are answering a multiple choice question. - \"\"\" - ) - - -Customizing prompts -^^^^^^^^^^^^^^^^^^^ -We can customize prompts by creating new classes that inherit from the `Prompt` class. -For example, consider the following custom agent instructions: + results = job.run() # This is equivalent to: results = survey.by(agent).by(model).run() -.. code-block:: python + # To select all the `prompt` columns at once: + # results.select("prompt.*").print(format="rich") - applicable_prompts = get_classes( - component_type="agent_instructions", - model=self.model.model, + # Or to specify the order in the table we can name them individually: + ( + results.select( + "q0_system_prompt", "q0_user_prompt", + "q1_system_prompt", "q1_user_prompt", + "q2_system_prompt", "q2_user_prompt" + ) + .print(format="rich") ) +Output: -Prompt class ------------- - -.. automodule:: edsl.prompts.Prompt - :members: - :undoc-members: - :show-inheritance: - - -Agent Instructions ------------------- - -.. automodule:: edsl.prompts.library.agent_instructions - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: edsl.prompts.library.agent_persona - :members: - :undoc-members: - :show-inheritance: - - -Question Instructions ---------------------- - -.. automodule:: edsl.prompts.library.question_multiple_choice - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: edsl.prompts.library.question_numerical - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: edsl.prompts.library.question_budget - :members: - :undoc-members: - :show-inheritance: - -.. automodule:: edsl.prompts.library.question_freetext - :members: - :undoc-members: - :show-inheritance: - +.. code-block:: text -QuestionInstructionBase class ------------------------------ + ┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓ + ┃ prompt ┃ prompt ┃ prompt ┃ prompt ┃ prompt ┃ prompt ┃ + ┃ .q0_system_prom… ┃ .q0_user_prompt ┃ .q1_system_prom… ┃ .q1_user_prompt ┃ .q2_system_prom… ┃ .q2_user_prompt ┃ + ┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩ + │ You are │ │ You are │ │ You are │ │ + │ answering │ Do you like │ answering │ Why not? │ answering │ Why? │ + │ questions as if │ school? │ questions as if │ │ questions as if │ │ + │ you were a │ │ you were a │ │ you were a │ │ + │ human. Do not │ │ human. Do not │ killer bees in │ human. Do not │ **lack*** of │ + │ break character. │ yes │ break character. │ cafeteria │ break character. │ killer bees in │ + │ You are an agent │ │ You are an agent │ │ You are an agent │ cafeteria │ + │ with the │ no │ with the │ other │ with the │ │ + │ following │ │ following │ │ following │ other │ + │ persona: │ │ persona: │ │ persona: │ │ + │ {'persona': │ Only 1 option │ {'persona': │ Only 1 option │ {'persona': │ │ + │ 'School │ may be selected. │ 'School │ may be selected. │ 'School │ Only 1 option │ + │ teacher'} │ │ teacher'} │ │ teacher'} │ may be selected. │ + │ │ Respond only │ │ Respond only │ │ │ + │ │ with a string │ │ with a string │ │ Respond only │ + │ │ corresponding to │ │ corresponding to │ │ with a string │ + │ │ one of the │ │ one of the │ │ corresponding to │ + │ │ options. │ │ options. │ │ one of the │ + │ │ │ │ │ │ options. │ + │ │ │ │ │ │ │ + │ │ After the │ │ After the │ │ │ + │ │ answer, you can │ │ answer, you can │ │ After the │ + │ │ put a comment │ │ put a comment │ │ answer, you can │ + │ │ explaining why │ │ explaining why │ │ put a comment │ + │ │ you chose that │ │ you chose that │ │ explaining why │ + │ │ option on the │ │ option on the │ │ you chose that │ + │ │ next line. │ │ next line. │ │ option on the │ + │ │ │ │ │ │ next line. │ + └──────────────────┴──────────────────┴──────────────────┴──────────────────┴──────────────────┴──────────────────┘ -.. automodule:: edsl.prompts.QuestionInstructionBase - :members: - :undoc-members: - :show-inheritance: \ No newline at end of file