diff --git a/rdagent/app/data_science/loop.py b/rdagent/app/data_science/loop.py index fafe4e53..2d53e616 100644 --- a/rdagent/app/data_science/loop.py +++ b/rdagent/app/data_science/loop.py @@ -21,6 +21,7 @@ Experiment2Feedback, ExpGen, Hypothesis2Experiment, + HypothesisFeedback, HypothesisGen, Trace, ) @@ -32,7 +33,7 @@ from rdagent.scenarios.data_science.experiment.experiment import DSExperiment from rdagent.scenarios.data_science.proposal.exp_gen import DSExpGen, DSTrace from rdagent.scenarios.kaggle.kaggle_crawler import download_data -from rdagent.core.proposal import HypothesisFeedback + class DataScienceRDLoop(RDLoop): skip_loop_error = (NextLoopException,) @@ -98,13 +99,19 @@ def running(self, prev_out: dict[str, Any]): def feedback(self, prev_out: dict[str, Any]): if not self.trace.all_components_completed(): - self.trace.hist.append((prev_out["direct_exp_gen"].hypothesis, prev_out["coding"], HypothesisFeedback( - observations="Not all 5 components are completed, skip feedback of DataScienceRDLoop.", - hypothesis_evaluation="", - new_hypothesis="", - reason="", - decision=True - ))) + self.trace.hist.append( + ( + prev_out["direct_exp_gen"].hypothesis, + prev_out["coding"], + HypothesisFeedback( + observations="Not all 5 components are completed, skip feedback of DataScienceRDLoop.", + hypothesis_evaluation="", + new_hypothesis="", + reason="", + decision=True, + ), + ) + ) raise NextLoopException("Not all 5 components are completed, skip feedback of DataScienceRDLoop.") feedback = self.summarizer.generate_feedback( diff --git a/rdagent/components/coder/data_science/feature/prompts.yaml b/rdagent/components/coder/data_science/feature/prompts.yaml index e003686c..8b63ea1a 100644 --- a/rdagent/components/coder/data_science/feature/prompts.yaml +++ b/rdagent/components/coder/data_science/feature/prompts.yaml @@ -1,6 +1,6 @@ feature: system: |- - You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science. + You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science. Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems. This project involves implementing feature engineering techniques to prepare data for machine learning models, and this project code will be written by GPT. @@ -33,7 +33,6 @@ feature: {% endfor %} {% endif %} - ``` user: |- ---------Feature Processing Specification--------- {{ feature_spec }} diff --git a/rdagent/components/coder/data_science/feature/test.py b/rdagent/components/coder/data_science/feature/test.py index 2f59f17f..c1f3e50b 100644 --- a/rdagent/components/coder/data_science/feature/test.py +++ b/rdagent/components/coder/data_science/feature/test.py @@ -20,7 +20,7 @@ def develop_one_competition(competition: str): # -> experiment feat_spec = file.read() # Create the experiment - ft = FeatureTask(name="FeatureTask", description=scen.competition_descriptions) + ft = FeatureTask(name="FeatureTask", description=scen.get_competition_full_desc()) exp = DSExperiment( sub_tasks=[ft], ) diff --git a/rdagent/components/coder/data_science/model/__init__.py b/rdagent/components/coder/data_science/model/__init__.py index 973741d1..f4bfbbd1 100644 --- a/rdagent/components/coder/data_science/model/__init__.py +++ b/rdagent/components/coder/data_science/model/__init__.py @@ -1,22 +1,88 @@ +import json +from pathlib import Path + +from jinja2 import Environment, StrictUndefined + from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator +from rdagent.components.coder.CoSTEER.evolving_strategy import ( + MultiProcessEvolvingStrategy, +) from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledge, ) -from rdagent.components.coder.data_science.model.es import ( - ModelMultiProcessEvolvingStrategy, -) from rdagent.components.coder.data_science.model.eval import ( ModelGeneralCaseSpecEvaluator, ) from rdagent.components.coder.data_science.model.exp import ModelTask +from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario +from rdagent.oai.llm_utils import APIBackend +from rdagent.utils.agent.tpl import T # from rdagent.utils.agent.tpl import T # T(".prompts:model_generator.user").r() +class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): + def implement_one_task( + self, + target_task: ModelTask, + queried_knowledge: CoSTEERQueriedKnowledge | None = None, + workspace: FBWorkspace | None = None, + ) -> dict[str, str]: + model_information_str = target_task.get_task_information() + + # 1. query + queried_similar_successful_knowledge = ( + queried_knowledge.task_to_similar_task_successful_knowledge[model_information_str] + if queried_knowledge is not None + else [] + ) + queried_former_failed_knowledge = ( + queried_knowledge.task_to_former_failed_traces[model_information_str] + if queried_knowledge is not None + else [] + ) + + # 2. code + system_prompt = T(".prompts:model_coder.system").r( + queried_similar_successful_knowledge=queried_similar_successful_knowledge, + queried_former_failed_knowledge=queried_former_failed_knowledge[0], + ) + user_prompt = T(".prompts:model_coder.user").r( + model_spec=workspace.code_dict["spec/model.md"], + latest_code=workspace.code_dict.get("model01.py"), + ) + + model_code = json.loads( + APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True + ) + )["code"] + + return { + "model01.py": model_code, + } + + def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo): + """ + Assign the code list to the evolving item. + + The code list is aligned with the evolving item's sub-tasks. + If a task is not implemented, put a None in the list. + """ + for index in range(len(evo.sub_tasks)): + if code_list[index] is None: + continue + if evo.sub_workspace_list[index] is None: + # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index]) + evo.sub_workspace_list[index] = evo.experiment_workspace + evo.sub_workspace_list[index].inject_code(**code_list[index]) + return evo + + class ModelCoSTEER(CoSTEER): def __init__( self, diff --git a/rdagent/components/coder/data_science/model/es.py b/rdagent/components/coder/data_science/model/es.py deleted file mode 100644 index 4408828d..00000000 --- a/rdagent/components/coder/data_science/model/es.py +++ /dev/null @@ -1,120 +0,0 @@ -import json -from pathlib import Path - -from jinja2 import Environment, StrictUndefined - -from rdagent.components.coder.CoSTEER.evolving_strategy import ( - MultiProcessEvolvingStrategy, -) -from rdagent.components.coder.CoSTEER.knowledge_management import ( - CoSTEERQueriedKnowledge, - CoSTEERQueriedKnowledgeV2, -) -from rdagent.components.coder.data_science.model.exp import ModelTask -from rdagent.core.experiment import FBWorkspace -from rdagent.core.prompts import Prompts -from rdagent.oai.llm_conf import LLM_SETTINGS -from rdagent.oai.llm_utils import APIBackend - -coder_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml") - - -class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): - def implement_one_task( - self, - target_task: ModelTask, - queried_knowledge: CoSTEERQueriedKnowledge | None = None, - workspace: FBWorkspace | None = None, - ) -> dict[str, str]: - model_information_str = target_task.get_task_information() - - queried_similar_successful_knowledge = ( - queried_knowledge.task_to_similar_task_successful_knowledge[model_information_str] - if queried_knowledge is not None - else [] - ) - queried_former_failed_knowledge = ( - queried_knowledge.task_to_former_failed_traces[model_information_str] - if queried_knowledge is not None - else [] - ) - - queried_former_failed_knowledge_to_render = ( - queried_former_failed_knowledge[0] - if isinstance(queried_knowledge, CoSTEERQueriedKnowledgeV2) - else queried_former_failed_knowledge - ) - - system_prompt = ( - Environment(undefined=StrictUndefined) - .from_string( - coder_prompts["model_coder"]["system"], - ) - .render( - # scenario=self.scen.get_scenario_all_desc(filtered_tag=target_task.model_type), - # TODO: fit new scenario information - spec=workspace.code_dict["spec/model.md"], - queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, - ) - ) - - queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge - for _ in range(10): # max attempt to reduce the length of user_prompt - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string( - coder_prompts["model_coder"]["user"], - ) - .render( - model_information_str=model_information_str, - queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render, - queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, - current_code=target_task.base_code, - ) - .strip("\n") - ) - if ( - APIBackend().build_messages_and_calculate_token( - user_prompt=user_prompt, - system_prompt=system_prompt, - ) - < LLM_SETTINGS.chat_token_limit - ): - break - elif len(queried_former_failed_knowledge_to_render) > 1: - queried_former_failed_knowledge_to_render = queried_former_failed_knowledge_to_render[1:] - elif len(queried_similar_successful_knowledge_to_render) > 1: - queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge_to_render[1:] - - model_code = json.loads( - # APIBackend(use_chat_cache=CoSTEER_SETTINGS.coder_use_cache).build_messages_and_create_chat_completion( - APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, - system_prompt=system_prompt, - json_mode=True, - ), - )["code"] - return { - "model01.py": model_code, - } - """ - import pandas as pd - def Model(): - pass - """ - - def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo): - """ - Assign the code list to the evolving item. - - The code list is aligned with the evolving item's sub-tasks. - If a task is not implemented, put a None in the list. - """ - for index in range(len(evo.sub_tasks)): - if code_list[index] is None: - continue - if evo.sub_workspace_list[index] is None: - # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index]) - evo.sub_workspace_list[index] = evo.experiment_workspace - evo.sub_workspace_list[index].inject_code(**code_list[index]) - return evo diff --git a/rdagent/components/coder/data_science/model/prompts.yaml b/rdagent/components/coder/data_science/model/prompts.yaml index 2185e6e0..c3b8c8f3 100644 --- a/rdagent/components/coder/data_science/model/prompts.yaml +++ b/rdagent/components/coder/data_science/model/prompts.yaml @@ -3,9 +3,6 @@ model_coder: You are tasked with implementing PyTorch models based on specific requirements provided by the user. The user’s ultimate goal is to obtain accurate predictions from the model on input data. Follow the instructions below to ensure your response is correct and aligned with the user’s expectations. Instructions for Code Generation: - Specification Compliance: - The user has provided a detailed framework or set of specifications under {{ spec }}. Your code must strictly adhere to this specification, including any required classes, methods, and organizational structure. Do not implement or add anything outside the scope of the provided specification. - Leveraging User Inputs: The user may provide various forms of additional information to guide you: @@ -26,39 +23,38 @@ model_coder: { "code": "Your corrected or newly implemented Python code as a single string" } - user: |- - Here is all the relevant information for this task: - - Target Model Details: - {{ model_information_str }} - + + -----------Here is the relevant information for this task----------- {% if queried_similar_successful_knowledge|length != 0 %} --------------Successful Implementations for Similar Models:-------------- - {% for similar_successful_knowledge in queried_similar_successful_knowledge %} - ===== Model {{loop.index}}: ===== + ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{loop.index}}:===== {{ similar_successful_knowledge.target_task.get_task_information() }} - ===== Code: ===== + =====Code:===== {{ similar_successful_knowledge.implementation.code }} {% endfor %} {% endif %} {% if queried_former_failed_knowledge|length != 0 %} --------------Previous Failed Attempts:-------------- - {% for former_failed_knowledge in queried_former_failed_knowledge %} - Attempt {{ loop.index }}: - ===== Code: ===== + {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}: + =====Code:===== {{ former_failed_knowledge.implementation.code }} - ===== Feedback: ===== + =====Feedback:===== {{ former_failed_knowledge.feedback }} {% endfor %} - {% endif %} + {% endif %} + + user: |- + ---------Model Specification--------- + {{ model_spec }} + + + {% if latest_code %} + ---------Former Specification--------- + Former Code: {{ latest_code }} + You should follow the former code to improve it. + {% endif %} - {% if current_code is not none %} - --------------Latest Code:-------------- - {{ current_code }} - {% else %} - No prior code has been implemented. - {% endif %} model_eval: system: |- diff --git a/rdagent/components/coder/data_science/model/test.py b/rdagent/components/coder/data_science/model/test.py index f3ced2bf..ba61d53a 100644 --- a/rdagent/components/coder/data_science/model/test.py +++ b/rdagent/components/coder/data_science/model/test.py @@ -6,9 +6,6 @@ from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS from rdagent.components.coder.data_science.model import ModelCoSTEER -from rdagent.components.coder.data_science.model.es import ( - ModelMultiProcessEvolvingStrategy, -) from rdagent.components.coder.data_science.model.eval import ( ModelGeneralCaseSpecEvaluator, ) diff --git a/rdagent/components/coder/data_science/workflow/__init__.py b/rdagent/components/coder/data_science/workflow/__init__.py index 8c120a7d..e590b05e 100644 --- a/rdagent/components/coder/data_science/workflow/__init__.py +++ b/rdagent/components/coder/data_science/workflow/__init__.py @@ -1,13 +1,82 @@ +import json + from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator -from rdagent.components.coder.data_science.workflow.es import ( - WorkflowMultiProcessEvolvingStrategy, +from rdagent.components.coder.CoSTEER.evolving_strategy import ( + MultiProcessEvolvingStrategy, +) +from rdagent.components.coder.CoSTEER.knowledge_management import ( + CoSTEERQueriedKnowledge, ) from rdagent.components.coder.data_science.workflow.eval import ( WorkflowGeneralCaseSpecEvaluator, ) +from rdagent.components.coder.data_science.workflow.exp import WorkflowTask +from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario +from rdagent.oai.llm_utils import APIBackend +from rdagent.utils.agent.tpl import T + + +class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): + def implement_one_task( + self, + target_task: WorkflowTask, + queried_knowledge: CoSTEERQueriedKnowledge | None = None, + workspace: FBWorkspace | None = None, + ) -> dict[str, str]: + # competition_info = self.scen.competition_descriptions + workflow_information_str = target_task.get_task_information() + + # 1. query + queried_similar_successful_knowledge = ( + queried_knowledge.task_to_similar_task_successful_knowledge[workflow_information_str] + if queried_knowledge is not None + else [] + ) + queried_former_failed_knowledge = ( + queried_knowledge.task_to_former_failed_traces[workflow_information_str] + if queried_knowledge is not None + else [] + ) + + # 2. code + system_prompt = T(".prompts:workflow_coder.system").r( + queried_similar_successful_knowledge=queried_similar_successful_knowledge, + queried_former_failed_knowledge=queried_former_failed_knowledge[0], + ) + user_prompt = T(".prompts:workflow_coder.user").r( + load_data_code=workspace.code_dict["load_data.py"], + feature_code=workspace.code_dict["feat01.py"], + model_code=workspace.code_dict["model01.py"], + ensemble_code=workspace.code_dict["ens.py"], + latest_code=workspace.code_dict.get("main.py"), + workflow_spec=workspace.code_dict["spec/workflow.md"], + ) + data_loader_code = json.loads( + APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True + ) + )["code"] + + return {"main.py": data_loader_code} + + def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo): + """ + Assign the code list to the evolving item. + + The code list is aligned with the evolving item's sub-tasks. + If a task is not implemented, put a None in the list. + """ + for index in range(len(evo.sub_tasks)): + if code_list[index] is None: + continue + if evo.sub_workspace_list[index] is None: + # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index]) + evo.sub_workspace_list[index] = evo.experiment_workspace + evo.sub_workspace_list[index].inject_code(**code_list[index]) + return evo class WorkflowCoSTEER(CoSTEER): diff --git a/rdagent/components/coder/data_science/workflow/es.py b/rdagent/components/coder/data_science/workflow/es.py deleted file mode 100644 index 378098cf..00000000 --- a/rdagent/components/coder/data_science/workflow/es.py +++ /dev/null @@ -1,53 +0,0 @@ -import json - -from rdagent.components.coder.CoSTEER.evolving_strategy import ( - MultiProcessEvolvingStrategy, -) -from rdagent.components.coder.CoSTEER.knowledge_management import ( - CoSTEERQueriedKnowledge, -) -from rdagent.components.coder.data_science.workflow.exp import WorkflowTask -from rdagent.core.experiment import FBWorkspace -from rdagent.oai.llm_utils import APIBackend -from rdagent.utils.agent.tpl import T - - -class WorkflowMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): - def implement_one_task( - self, - target_task: WorkflowTask, - queried_knowledge: CoSTEERQueriedKnowledge | None = None, - workspace: FBWorkspace | None = None, - ) -> dict[str, str]: - # competition_info = self.scen.competition_descriptions - - system_prompt = T(".prompts:workflow_coder.system").r(workflow_spec=workspace.code_dict["spec/workflow.md"]) - user_prompt = T(".prompts:workflow_coder.user").r( - load_data_code=workspace.code_dict["load_data.py"], - feature_code=workspace.code_dict["feat01.py"], - model_code=workspace.code_dict["model01.py"], - ensemble_code=workspace.code_dict["ens.py"], - ) - data_loader_code = json.loads( - APIBackend().build_messages_and_create_chat_completion( - user_prompt=user_prompt, system_prompt=system_prompt, json_mode=True - ) - )["code"] - - return {"main.py": data_loader_code} - - def assign_code_list_to_evo(self, code_list: list[dict[str, str]], evo): - """ - Assign the code list to the evolving item. - - The code list is aligned with the evolving item's sub-tasks. - If a task is not implemented, put a None in the list. - """ - for index in range(len(evo.sub_tasks)): - if code_list[index] is None: - continue - if evo.sub_workspace_list[index] is None: - # evo.sub_workspace_list[index] = FBWorkspace(target_task=evo.sub_tasks[index]) - evo.sub_workspace_list[index] = evo.experiment_workspace - evo.sub_workspace_list[index].inject_code(**code_list[index]) - return evo diff --git a/rdagent/components/coder/data_science/workflow/exp.py b/rdagent/components/coder/data_science/workflow/exp.py index f2934ee6..0c4124ed 100644 --- a/rdagent/components/coder/data_science/workflow/exp.py +++ b/rdagent/components/coder/data_science/workflow/exp.py @@ -24,3 +24,8 @@ def from_dict(dict): def __repr__(self) -> str: return f"<{self.__class__.__name__} {self.name}>" + + def get_task_information(self): + return f"""name: {self.name} +description: {self.description} +""" diff --git a/rdagent/components/coder/data_science/workflow/prompts.yaml b/rdagent/components/coder/data_science/workflow/prompts.yaml index 4937a4bc..c0214d6a 100644 --- a/rdagent/components/coder/data_science/workflow/prompts.yaml +++ b/rdagent/components/coder/data_science/workflow/prompts.yaml @@ -1,7 +1,8 @@ workflow_coder: system: |- - You are a Python data scientist working on a new Kaggle competition project. - + You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science. + Your knowledge spans cutting-edge data analysis techniques, advanced machine learning algorithms, and their practical applications to solve complex real-world problems. + The user has written different Python functions that can load and preprocess data, execute feature engineering, train models, and ensemble them. These Python codes with different functionalities are written separately in different Python files. @@ -9,11 +10,6 @@ workflow_coder: This workflow code is also a Python file, and it functions similarly to a main process that calls the sub-files for each step and ultimately outputs a prediction file. The user will also provide specifications on how to organize the code and give instructions. - These specifications are as follows: - {{ workflow_spec }} - - The dataset provided by load_data is not split into training and testing sets. In the workflow, you should perform this splitting. - By default, use 80% of the data for training and 20% for testing. If the specification requires a different split ratio, cross-validation, or other splitting methods, follow the specification. The code you implement should align with the framework given in the specifications. After predicting the output, print the shape and other information of the output to stdout to help the evaluator assess the code. @@ -23,7 +19,30 @@ workflow_coder: "code": "The Python code as a string." } + -----------Here is the relevant information for this task----------- + {% if queried_similar_successful_knowledge|length != 0 %} + --------------Successful Implementations for Similar Models:-------------- + ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{loop.index}}:===== + {{ similar_successful_knowledge.target_task.get_task_information() }} + =====Code:===== + {{ similar_successful_knowledge.implementation.code }} + {% endfor %} + {% endif %} + + {% if queried_former_failed_knowledge|length != 0 %} + --------------Previous Failed Attempts:-------------- + {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}: + =====Code:===== + {{ former_failed_knowledge.implementation.code }} + =====Feedback:===== + {{ former_failed_knowledge.feedback }} + {% endfor %} + {% endif %} + user: |- + ---------Workflow Specification--------- + {{ workflow_spec }} + ---------load data code--------- file: load_data.py {{ load_data_code }} @@ -41,6 +60,12 @@ workflow_coder: file: ens.py {{ ensemble_code }} + {% if latest_code %} + ---------Former Specification--------- + Former Code: {{ latest_code }} + You should follow the former code to improve it. + {% endif %} + workflow_eval: system: |- You are a data scientist. diff --git a/rdagent/components/coder/data_science/workflow/test.py b/rdagent/components/coder/data_science/workflow/test.py index d6e255c1..c8e6eb3f 100644 --- a/rdagent/components/coder/data_science/workflow/test.py +++ b/rdagent/components/coder/data_science/workflow/test.py @@ -6,9 +6,6 @@ from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER -from rdagent.components.coder.data_science.workflow.es import ( - WorkflowMultiProcessEvolvingStrategy, -) from rdagent.components.coder.data_science.workflow.eval import ( WorkflowGeneralCaseSpecEvaluator, )