From bf4995b46ea638bc783d764d064032f24752387a Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 00:14:17 +0800 Subject: [PATCH 01/11] fix isort & black & toml-sort & sphinx error --- .github/workflows/ci.yml | 4 +- Makefile | 2 +- docs/conf.py | 2 +- docs/scenarios_and_quickstart.rst | 4 +- pyproject.toml | 7 +- rdagent/app/data_mining/conf.py | 7 +- rdagent/app/data_mining/model.py | 2 + rdagent/app/qlib_rd_loop/RDAgent.py | 30 +- .../app/qlib_rd_loop/factor_from_report.py | 53 ++- .../app/qlib_rd_loop/factor_from_report_sh.py | 62 +-- rdagent/app/qlib_rd_loop/factor_w_sc.py | 1 - .../app/quant_factor_benchmark/analysis.py | 29 +- rdagent/app/quant_factor_benchmark/eval.py | 21 +- rdagent/components/benchmark/eval_method.py | 4 +- .../coder/factor_coder/CoSTEER/evaluators.py | 14 +- .../factor_coder/CoSTEER/evolving_strategy.py | 8 +- .../components/coder/factor_coder/factor.py | 6 +- rdagent/components/coder/model_coder/model.py | 23 +- .../document_reader/document_reader.py | 10 +- rdagent/components/proposal/model_proposal.py | 2 +- rdagent/components/workflow/conf.py | 2 + rdagent/components/workflow/rd_loop.py | 6 +- rdagent/core/evolving_agent.py | 11 +- rdagent/core/proposal.py | 10 +- rdagent/core/utils.py | 2 +- rdagent/log/base.py | 12 +- rdagent/log/logger.py | 6 +- rdagent/log/storage.py | 17 +- rdagent/log/ui/app.py | 14 +- rdagent/log/ui/st_fixed_container.py | 2 +- rdagent/log/ui/web.py | 408 ++++++++++-------- rdagent/log/utils.py | 3 +- .../data_mining/developer/feedback.py | 2 +- .../data_mining/developer/model_runner.py | 1 - .../experiment/model_experiment.py | 8 +- .../experiment/model_template/train.py | 42 +- .../data_mining/experiment/workspace.py | 5 +- .../data_mining/proposal/model_proposal.py | 7 +- rdagent/scenarios/qlib/developer/feedback.py | 39 +- .../qlib/experiment/factor_experiment.py | 4 +- .../factor_template/read_exp_res.py | 2 +- .../qlib/experiment/model_experiment.py | 8 +- .../experiment/model_template/read_exp_res.py | 1 - .../factor_experiment_loader/json_loader.py | 4 +- .../factor_experiment_loader/pdf_loader.py | 13 +- .../scenarios/qlib/proposal/model_proposal.py | 12 +- rdagent/utils/agent/ret.py | 2 +- rdagent/utils/agent/tpl.py | 20 +- rdagent/utils/env.py | 21 +- rdagent/utils/workflow.py | 25 +- test/utils/test_agent_infra.py | 4 - test/utils/test_misc.py | 1 - 52 files changed, 550 insertions(+), 455 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f47df5cce..43b67d49a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,9 +22,9 @@ jobs: - env: CHAT_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }} CHAT_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} - CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }} + # CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }} CHAT_MODEL: ${{ secrets.CHAT_MODEL }} - CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }} + # CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }} EMBEDDING_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }} EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }} diff --git a/Makefile b/Makefile index 2f1ce9d3e..e9a51dd2f 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ toml-sort: # Check lint with all linters. # lint: black isort mypy ruff toml-sort -lint: mypy ruff +lint: mypy ruff isort black toml-sort sphinx # Run pre-commit with autofix against all files. pre-commit: diff --git a/docs/conf.py b/docs/conf.py index 426a57cd4..65fb93886 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - 'sphinx.ext.autodoc', + "sphinx.ext.autodoc", ] autodoc_member_order = "bysource" diff --git a/docs/scenarios_and_quickstart.rst b/docs/scenarios_and_quickstart.rst index e264cb242..5969d94b3 100644 --- a/docs/scenarios_and_quickstart.rst +++ b/docs/scenarios_and_quickstart.rst @@ -23,7 +23,7 @@ Scenario lists - Scnarios' demo & quick start -========================= +============================ Scen1 ----- @@ -69,7 +69,7 @@ TODO: Show some examples: Scen2: ------ +------ 📄 Research Report-Based Factor Extraction Scen2 Intro diff --git a/pyproject.toml b/pyproject.toml index 40ced3626..209ad958c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "setuptools", "setuptools-scm", ] -root= "rdagent" +root = "rdagent" [project] authors = [ @@ -27,14 +27,13 @@ dynamic = [ ] keywords = [ "Autonomous Agents", - "Research and Development", "Large Language Models", + "Research and Development", ] name = "rdagent" readme = "README.md" requires-python = ">=3.8" - [project.urls] homepage = "https://github.com/microsoft/RD-Agent/" issue = "https://github.com/microsoft/RD-Agent/issues" @@ -52,13 +51,13 @@ color_output = true profile = "black" [tool.mypy] -explicit_package_bases = true check_untyped_defs = true disallow_any_unimported = true disallow_untyped_defs = true enable_error_code = [ "ignore-without-code", ] +explicit_package_bases = true warn_return_any = true warn_unused_ignores = true diff --git a/rdagent/app/data_mining/conf.py b/rdagent/app/data_mining/conf.py index df2280ff4..1e54faedc 100644 --- a/rdagent/app/data_mining/conf.py +++ b/rdagent/app/data_mining/conf.py @@ -1,6 +1,5 @@ from pathlib import Path - from rdagent.components.workflow.conf import BasePropSetting @@ -19,11 +18,11 @@ class Config: evolving_n: int = 10 - # 2) Extra config for the scenario + # 2) Extra config for the scenario # physionet account # NOTE: You should apply the account in https://physionet.org/ - username: str = '' - password: str = '' + username: str = "" + password: str = "" PROP_SETTING = PropSetting() diff --git a/rdagent/app/data_mining/model.py b/rdagent/app/data_mining/model.py index 83b83459b..1fb56db35 100644 --- a/rdagent/app/data_mining/model.py +++ b/rdagent/app/data_mining/model.py @@ -1,8 +1,10 @@ import fire + from rdagent.app.data_mining.conf import PROP_SETTING from rdagent.components.workflow.rd_loop import RDLoop from rdagent.core.exception import ModelEmptyError + class ModelRDLoop(RDLoop): skip_loop_error = (ModelEmptyError,) diff --git a/rdagent/app/qlib_rd_loop/RDAgent.py b/rdagent/app/qlib_rd_loop/RDAgent.py index 3cc2f9bdd..73571a504 100644 --- a/rdagent/app/qlib_rd_loop/RDAgent.py +++ b/rdagent/app/qlib_rd_loop/RDAgent.py @@ -1,4 +1,5 @@ import pickle + from rdagent.app.qlib_rd_loop.conf import PROP_SETTING from rdagent.core.developer import Developer from rdagent.core.exception import ModelEmptyError @@ -12,6 +13,7 @@ from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger + # TODO: we can design a workflow that can automatically save session and traceback in the future class Model_RD_Agent: def __init__(self): @@ -20,50 +22,55 @@ def __init__(self): self.hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.model_hypothesis2experiment)() self.qlib_model_coder: Developer = import_class(PROP_SETTING.model_coder)(self.scen) self.qlib_model_runner: Developer = import_class(PROP_SETTING.model_runner)(self.scen) - self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)(self.scen) + self.qlib_model_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.model_summarizer)( + self.scen + ) self.trace = Trace(scen=self.scen) def generate_hypothesis(self): hypothesis = self.hypothesis_gen.gen(self.trace) - self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename='step_hypothesis.pkl') + self.dump_objects(hypothesis=hypothesis, trace=self.trace, filename="step_hypothesis.pkl") return hypothesis def convert_hypothesis(self, hypothesis): exp = self.hypothesis2experiment.convert(hypothesis, self.trace) - self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename='step_experiment.pkl') + self.dump_objects(exp=exp, hypothesis=hypothesis, trace=self.trace, filename="step_experiment.pkl") return exp def generate_code(self, exp): exp = self.qlib_model_coder.develop(exp) - self.dump_objects(exp=exp, trace=self.trace, filename='step_code.pkl') + self.dump_objects(exp=exp, trace=self.trace, filename="step_code.pkl") return exp def run_experiment(self, exp): exp = self.qlib_model_runner.develop(exp) - self.dump_objects(exp=exp, trace=self.trace, filename='step_run.pkl') + self.dump_objects(exp=exp, trace=self.trace, filename="step_run.pkl") return exp def generate_feedback(self, exp, hypothesis): feedback = self.qlib_model_summarizer.generate_feedback(exp, hypothesis, self.trace) - self.dump_objects(exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename='step_feedback.pkl') + self.dump_objects( + exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename="step_feedback.pkl" + ) return feedback def append_to_trace(self, hypothesis, exp, feedback): self.trace.hist.append((hypothesis, exp, feedback)) - self.dump_objects(trace=self.trace, filename='step_trace.pkl') + self.dump_objects(trace=self.trace, filename="step_trace.pkl") - def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename='dumped_objects.pkl'): - with open(filename, 'wb') as f: + def dump_objects(self, exp=None, hypothesis=None, feedback=None, trace=None, filename="dumped_objects.pkl"): + with open(filename, "wb") as f: pickle.dump((exp, hypothesis, feedback, trace or self.trace), f) def load_objects(self, filename): - with open(filename, 'rb') as f: + with open(filename, "rb") as f: return pickle.load(f) + def process_steps(agent): # Load trace if available try: - _, _, _, trace = agent.load_objects('step_trace.pkl') + _, _, _, trace = agent.load_objects("step_trace.pkl") agent.trace = trace print(trace.get_sota_hypothesis_and_experiment()) except FileNotFoundError: @@ -99,6 +106,7 @@ def process_steps(agent): # # Step 6: Append to trace # agent.append_to_trace(hypothesis, exp, feedback) + if __name__ == "__main__": agent = Model_RD_Agent() process_steps(agent) diff --git a/rdagent/app/qlib_rd_loop/factor_from_report.py b/rdagent/app/qlib_rd_loop/factor_from_report.py index 4ea9410a1..281f96060 100644 --- a/rdagent/app/qlib_rd_loop/factor_from_report.py +++ b/rdagent/app/qlib_rd_loop/factor_from_report.py @@ -1,34 +1,38 @@ import json -from pathlib import Path import pickle +from pathlib import Path + +import pandas as pd from dotenv import load_dotenv from jinja2 import Environment, StrictUndefined -import pandas as pd from rdagent.app.qlib_rd_loop.conf import PROP_SETTING -from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain +from rdagent.components.document_reader.document_reader import ( + load_and_process_pdfs_by_langchain, +) +from rdagent.core.developer import Developer from rdagent.core.prompts import Prompts +from rdagent.core.proposal import ( + Hypothesis, + Hypothesis2Experiment, + HypothesisExperiment2Feedback, + HypothesisGen, + Trace, +) from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER -from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment +from rdagent.scenarios.qlib.experiment.factor_experiment import ( + QlibFactorExperiment, + QlibFactorScenario, +) from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( FactorExperimentLoaderFromPDFfiles, classify_report_from_dict, ) -from rdagent.core.proposal import ( - Hypothesis2Experiment, - HypothesisExperiment2Feedback, - HypothesisGen, - Hypothesis, - Trace, -) - -from rdagent.core.developer import Developer - assert load_dotenv() scen: Scenario = import_class(PROP_SETTING.factor_scen)() @@ -43,17 +47,21 @@ qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) -with open(PROP_SETTING.report_result_json_file_path, 'r') as f: +with open(PROP_SETTING.report_result_json_file_path, "r") as f: judge_pdf_data = json.load(f) prompts_path = Path(__file__).parent / "prompts.yaml" prompts = Prompts(file_path=prompts_path) + def generate_hypothesis(factor_result: dict, report_content: str) -> str: - system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() - user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render( - factor_descriptions=json.dumps(factor_result), - report_content=report_content + system_prompt = ( + Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() + ) + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompts["hypothesis_generation"]["user"]) + .render(factor_descriptions=json.dumps(factor_result), report_content=report_content) ) response = APIBackend().build_messages_and_create_chat_completion( @@ -68,16 +76,16 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str: return Hypothesis(hypothesis=hypothesis_text, reason=reason_text) + def extract_factors_and_implement(report_file_path: str) -> tuple: scenario = QlibFactorScenario() with logger.tag("extract_factors_and_implement"): with logger.tag("load_factor_tasks"): - exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) if exp is None or exp.sub_tasks == []: return None, None - + docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path)) factor_result = { @@ -85,7 +93,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: "description": task.factor_description, "formulation": task.factor_formulation, "variables": task.variables, - "resources": task.factor_resources + "resources": task.factor_resources, } for task in exp.sub_tasks } @@ -95,6 +103,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: return exp, hypothesis + trace = Trace(scen=scen) for file_path, attributes in judge_pdf_data.items(): diff --git a/rdagent/app/qlib_rd_loop/factor_from_report_sh.py b/rdagent/app/qlib_rd_loop/factor_from_report_sh.py index c8cc57e4a..f0d0a961e 100644 --- a/rdagent/app/qlib_rd_loop/factor_from_report_sh.py +++ b/rdagent/app/qlib_rd_loop/factor_from_report_sh.py @@ -1,35 +1,40 @@ # TODO: we should have more advanced mechanism to handle such requirements for saving sessions. import json -from pathlib import Path import pickle +from pathlib import Path + +import pandas as pd from dotenv import load_dotenv from jinja2 import Environment, StrictUndefined -import pandas as pd from rdagent.app.qlib_rd_loop.conf import PROP_SETTING -from rdagent.components.document_reader.document_reader import extract_first_page_screenshot_from_pdf, load_and_process_pdfs_by_langchain +from rdagent.components.document_reader.document_reader import ( + extract_first_page_screenshot_from_pdf, + load_and_process_pdfs_by_langchain, +) +from rdagent.core.developer import Developer from rdagent.core.prompts import Prompts +from rdagent.core.proposal import ( + Hypothesis, + Hypothesis2Experiment, + HypothesisExperiment2Feedback, + HypothesisGen, + Trace, +) from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER -from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment +from rdagent.scenarios.qlib.experiment.factor_experiment import ( + QlibFactorExperiment, + QlibFactorScenario, +) from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( FactorExperimentLoaderFromPDFfiles, classify_report_from_dict, ) -from rdagent.core.proposal import ( - Hypothesis2Experiment, - HypothesisExperiment2Feedback, - HypothesisGen, - Hypothesis, - Trace, -) - -from rdagent.core.developer import Developer - assert load_dotenv() scen: Scenario = import_class(PROP_SETTING.factor_scen)() @@ -44,27 +49,33 @@ qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) -with open(PROP_SETTING.report_result_json_file_path, 'r') as f: +with open(PROP_SETTING.report_result_json_file_path, "r") as f: judge_pdf_data = json.load(f) prompts_path = Path(__file__).parent / "prompts.yaml" prompts = Prompts(file_path=prompts_path) + def save_progress(trace, current_index): with open(PROP_SETTING.progress_file_path, "wb") as f: pickle.dump((trace, current_index), f) + def load_progress(): if Path(PROP_SETTING.progress_file_path).exists(): with open(PROP_SETTING.progress_file_path, "rb") as f: return pickle.load(f) return Trace(scen=scen), 0 + def generate_hypothesis(factor_result: dict, report_content: str) -> str: - system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() - user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render( - factor_descriptions=json.dumps(factor_result), - report_content=report_content + system_prompt = ( + Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() + ) + user_prompt = ( + Environment(undefined=StrictUndefined) + .from_string(prompts["hypothesis_generation"]["user"]) + .render(factor_descriptions=json.dumps(factor_result), report_content=report_content) ) response = APIBackend().build_messages_and_create_chat_completion( @@ -79,12 +90,12 @@ def generate_hypothesis(factor_result: dict, report_content: str) -> str: return Hypothesis(hypothesis=hypothesis_text, reason=reason_text) + def extract_factors_and_implement(report_file_path: str) -> tuple: scenario = QlibFactorScenario() with logger.tag("extract_factors_and_implement"): with logger.tag("load_factor_tasks"): - exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) if exp is None or exp.sub_tasks == []: return None, None @@ -100,7 +111,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: "description": task.factor_description, "formulation": task.factor_formulation, "variables": task.variables, - "resources": task.factor_resources + "resources": task.factor_resources, } for task in exp.sub_tasks } @@ -110,6 +121,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: return exp, hypothesis + trace, start_index = load_progress() try: @@ -122,7 +134,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path)) if report_file_path.exists(): logger.info(f"Processing {report_file_path}") - + with logger.tag("r"): exp, hypothesis = extract_factors_and_implement(str(report_file_path)) if exp is None: @@ -132,7 +144,7 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[])) logger.log_object(hypothesis, tag="hypothesis generation") logger.log_object(exp.sub_tasks, tag="experiment generation") - + with logger.tag("d"): exp = qlib_factor_coder.develop(exp) logger.log_object(exp.sub_workspace_list) @@ -145,10 +157,10 @@ def extract_factors_and_implement(report_file_path: str) -> tuple: logger.log_object(exp, tag="factor runner result") feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace) logger.log_object(feedback, tag="feedback") - + trace.hist.append((hypothesis, exp, feedback)) logger.info(f"Processed {report_file_path}: Result: {exp}") - + # Save progress after processing each report save_progress(trace, index + 1) else: diff --git a/rdagent/app/qlib_rd_loop/factor_w_sc.py b/rdagent/app/qlib_rd_loop/factor_w_sc.py index 1e6f82202..31fc0c1ac 100644 --- a/rdagent/app/qlib_rd_loop/factor_w_sc.py +++ b/rdagent/app/qlib_rd_loop/factor_w_sc.py @@ -33,6 +33,5 @@ def main(path=None, step_n=None): fire.Fire(main) - if __name__ == "__main__": fire.Fire(main) diff --git a/rdagent/app/quant_factor_benchmark/analysis.py b/rdagent/app/quant_factor_benchmark/analysis.py index 75aea00db..6167ab678 100644 --- a/rdagent/app/quant_factor_benchmark/analysis.py +++ b/rdagent/app/quant_factor_benchmark/analysis.py @@ -1,12 +1,14 @@ import json import pickle -import pandas as pd from pathlib import Path + import matplotlib.pyplot as plt +import pandas as pd import seaborn as sns -from rdagent.components.benchmark.eval_method import FactorImplementEval from rdagent.components.benchmark.conf import BenchmarkSettings +from rdagent.components.benchmark.eval_method import FactorImplementEval + class BenchmarkAnalyzer: def __init__(self, settings): @@ -25,10 +27,10 @@ def load_data(self, file_path): file_path = Path(file_path) if not (file_path.is_file() and file_path.suffix == ".pkl"): raise ValueError("Invalid file path") - + with file_path.open("rb") as f: res = pickle.load(f) - + return res def process_results(self, results): @@ -39,7 +41,7 @@ def process_results(self, results): processed_data = self.analyze_data(summarized_data) final_res[experiment] = processed_data.iloc[-1, :] return final_res - + def reformat_succ_rate(self, display_df): new_idx = [] display_df = display_df[display_df.index.isin(self.index_map.keys())] @@ -52,8 +54,10 @@ def reformat_succ_rate(self, display_df): ) display_df = display_df.swaplevel(0, 2).swaplevel(0, 1).sort_index(axis=0) - return display_df.sort_index(key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x]) - + return display_df.sort_index( + key=lambda x: [{"Easy": 0, "Medium": 1, "Hard": 2, "New Discovery": 3}.get(i, i) for i in x] + ) + def result_all_key_order(self, x): order_v = [] for i in x: @@ -92,9 +96,7 @@ def analyze_data(self, sum_df): sum_df_clean["FactorRowCountEvaluator"] - format_issue = ( - sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"] - ) + format_issue = sum_df_clean["FactorRowCountEvaluator"] & sum_df_clean["FactorIndexEvaluator"] eval_series = format_issue.unstack() succ_rate = eval_series.T.fillna(False).astype(bool) # false indicate failure format_succ_rate = succ_rate.mean(axis=0).to_frame("success rate") @@ -113,10 +115,7 @@ def analyze_data(self, sum_df): value_max_res = self.reformat_succ_rate(value_max) value_avg = ( - (sum_df_clean["FactorMissingValuesEvaluator"] * format_issue) - .unstack() - .T.mean(axis=0) - .to_frame("avg_value") + (sum_df_clean["FactorMissingValuesEvaluator"] * format_issue).unstack().T.mean(axis=0).to_frame("avg_value") ) value_avg_res = self.reformat_succ_rate(value_avg) @@ -148,7 +147,6 @@ def analyze_data(self, sum_df): return df_w_mean - class Plotter: @staticmethod def change_fs(font_size): @@ -169,6 +167,7 @@ def plot_data(data, file_name): plt.title("Comparison of Different Methods") plt.savefig(file_name) + if __name__ == "__main__": settings = BenchmarkSettings() benchmark = BenchmarkAnalyzer(settings) diff --git a/rdagent/app/quant_factor_benchmark/eval.py b/rdagent/app/quant_factor_benchmark/eval.py index a22f928fc..2f3cf2752 100644 --- a/rdagent/app/quant_factor_benchmark/eval.py +++ b/rdagent/app/quant_factor_benchmark/eval.py @@ -1,22 +1,19 @@ import os -from pathlib import Path import pickle import time -from rdagent.app.qlib_rd_loop.conf import PROP_SETTING -from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import ( - FactorTestCaseLoaderFromJsonFile, -) +from pathlib import Path +from pprint import pprint +from rdagent.app.qlib_rd_loop.conf import PROP_SETTING from rdagent.components.benchmark.conf import BenchmarkSettings from rdagent.components.benchmark.eval_method import FactorImplementEval -from rdagent.core.utils import import_class - -from rdagent.core.utils import import_class from rdagent.core.scenario import Scenario +from rdagent.core.utils import import_class +from rdagent.log import rdagent_logger as logger from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario - -from pprint import pprint +from rdagent.scenarios.qlib.factor_experiment_loader.json_loader import ( + FactorTestCaseLoaderFromJsonFile, +) # 1.read the settings bs = BenchmarkSettings() @@ -28,7 +25,7 @@ scen: Scenario = import_class(PROP_SETTING.factor_scen)() generate_method = import_class(bs.bench_method_cls)(scen=scen) - + # 4.declare the eval method and pass the arguments. eval_method = FactorImplementEval( method=generate_method, diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index c8f2aec84..b1b4632d3 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -19,18 +19,17 @@ from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.developer import Developer - from rdagent.core.exception import CoderException, RunnerException from rdagent.core.experiment import Task, Workspace from rdagent.core.scenario import Scenario from rdagent.core.utils import multiprocessing_wrapper - EVAL_RES = Dict[ str, List[Tuple[FactorEvaluator, Union[object, RunnerException]]], ] + class TestCase: def __init__( self, @@ -197,4 +196,3 @@ def summarize_res(res: EVAL_RES) -> pd.DataFrame: sum_res[key] = val return pd.DataFrame(sum_res) - diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index a1ddf6f7c..bd732bcb2 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -356,7 +356,9 @@ def evaluate( # Check if both dataframe have the same rows count if gt_implementation is not None: - feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, single_column_result = FactorRowCountEvaluator(self.scen).evaluate( + implementation, gt_implementation + ) conclusions.append(feedback_str) feedback_str, same_index_result = FactorIndexEvaluator(self.scen).evaluate( @@ -364,7 +366,9 @@ def evaluate( ) conclusions.append(feedback_str) - feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate( + implementation, gt_implementation + ) conclusions.append(feedback_str) feedback_str, equal_value_ratio_result = FactorEqualValueCountEvaluator(self.scen).evaluate( @@ -464,8 +468,10 @@ def evaluate( except KeyError as e: attempts += 1 if attempts >= max_attempts: - raise KeyError("Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts.") from e - + raise KeyError( + "Response from API is missing 'final_decision' or 'final_feedback' key after multiple attempts." + ) from e + return None, None diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index 1493cfac8..b5a66e5f5 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -68,8 +68,12 @@ def evolve( # if the number of factors to be implemented is larger than the limit, we need to select some of them if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1: # if the number of loops is equal to the select_loop, we need to select some of them - implementation_factors_per_round = round(FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5) # ceilling - implementation_factors_per_round = min(implementation_factors_per_round, len(to_be_finished_task_index)) # but not exceed the total number of tasks + implementation_factors_per_round = round( + FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5 + ) # ceilling + implementation_factors_per_round = min( + implementation_factors_per_round, len(to_be_finished_task_index) + ) # but not exceed the total number of tasks if FACTOR_IMPLEMENT_SETTINGS.select_method == "random": to_be_finished_task_index = RandomSelect( diff --git a/rdagent/components/coder/factor_coder/factor.py b/rdagent/components/coder/factor_coder/factor.py index 9dd51466c..412f98a66 100644 --- a/rdagent/components/coder/factor_coder/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -10,11 +10,7 @@ from filelock import FileLock from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS -from rdagent.core.exception import ( - CodeFormatError, - NoOutputError, - CustomRuntimeError, -) +from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError from rdagent.core.experiment import Experiment, FBWorkspace, Task from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import md5_hash diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index eeff3d7c9..7d6801851 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -16,7 +16,14 @@ class ModelTask(Task): def __init__( - self, name: str, description: str, formulation: str, architecture: str, variables: Dict[str, str], hyperparameters: Dict[str, str], model_type: Optional[str] = None + self, + name: str, + description: str, + formulation: str, + architecture: str, + variables: Dict[str, str], + hyperparameters: Dict[str, str], + model_type: Optional[str] = None, ) -> None: self.name: str = name self.description: str = description @@ -24,7 +31,9 @@ def __init__( self.architecture: str = architecture self.variables: str = variables self.hyperparameters: str = hyperparameters - self.model_type: str = model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model + self.model_type: str = ( + model_type # Tabular for tabular model, TimesSeries for time series model, Graph for graph model + ) def get_task_information(self): return f"""name: {self.name} @@ -107,21 +116,21 @@ def execute( # Initialize all parameters of `m` to `param_init_value` for _, param in m.named_parameters(): param.data.fill_(param_init_value) - + # Execute the model if self.target_task.model_type == "Graph": out = m(*data) else: out = m(data) - + execution_model_output = out.cpu().detach() execution_feedback_str = f"Execution successful, output tensor shape: {execution_model_output.shape}" - + if MODEL_IMPL_SETTINGS.enable_execution_cache: pickle.dump((execution_feedback_str, execution_model_output), open(cache_file_path, "wb")) - + return execution_feedback_str, execution_model_output - + except Exception as e: return f"Execution error: {e}", None diff --git a/rdagent/components/document_reader/document_reader.py b/rdagent/components/document_reader/document_reader.py index 9341066f9..d61476916 100644 --- a/rdagent/components/document_reader/document_reader.py +++ b/rdagent/components/document_reader/document_reader.py @@ -1,16 +1,17 @@ from __future__ import annotations from pathlib import Path -import fitz -from PIL import Image from typing import TYPE_CHECKING +import fitz from azure.ai.formrecognizer import DocumentAnalysisClient from azure.core.credentials import AzureKeyCredential from langchain.document_loaders import PyPDFDirectoryLoader, PyPDFLoader +from PIL import Image if TYPE_CHECKING: from langchain_core.documents import Document + from rdagent.core.conf import RD_AGENT_SETTINGS @@ -99,10 +100,11 @@ def load_and_process_pdfs_by_azure_document_intelligence(path: Path) -> dict[str ) return content_dict + def extract_first_page_screenshot_from_pdf(pdf_path: Path) -> Image: doc = fitz.open(pdf_path) page = doc.load_page(0) pix = page.get_pixmap() image = Image.frombytes("RGB", [pix.width, pix.height], pix.samples) - - return image \ No newline at end of file + + return image diff --git a/rdagent/components/proposal/model_proposal.py b/rdagent/components/proposal/model_proposal.py index c6b59a03b..a5d280069 100644 --- a/rdagent/components/proposal/model_proposal.py +++ b/rdagent/components/proposal/model_proposal.py @@ -15,11 +15,11 @@ ) from rdagent.oai.llm_utils import APIBackend - ModelHypothesis = Hypothesis prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml") + class ModelHypothesisGen(HypothesisGen): prompts: Prompts = prompt_dict diff --git a/rdagent/components/workflow/conf.py b/rdagent/components/workflow/conf.py index fc8f84ad7..054c63cec 100644 --- a/rdagent/components/workflow/conf.py +++ b/rdagent/components/workflow/conf.py @@ -1,5 +1,6 @@ from pydantic_settings import BaseSettings + class BasePropSetting(BaseSettings): """ The common part of the config for RD Loop to propose and developement @@ -11,6 +12,7 @@ class Config: env_prefix = "DM_MODEL_" # Use MODEL_CODER_ as prefix for environment variables protected_namespaces = () # Add 'model_' to the protected namespaces """ + scen: str = "" hypothesis_gen: str = "" hypothesis2experiment: str = "" diff --git a/rdagent/components/workflow/rd_loop.py b/rdagent/components/workflow/rd_loop.py index d1a400173..dcb1cbdc7 100644 --- a/rdagent/components/workflow/rd_loop.py +++ b/rdagent/components/workflow/rd_loop.py @@ -4,6 +4,7 @@ """ from typing import Any + from rdagent.components.workflow.conf import BasePropSetting from rdagent.core.developer import Developer from rdagent.core.proposal import ( @@ -15,11 +16,10 @@ from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger +from rdagent.utils.workflow import LoopBase, LoopMeta -from rdagent.utils.workflow import LoopMeta, LoopBase class RDLoop(LoopBase, metaclass=LoopMeta): - def __init__(self, PROP_SETTING: BasePropSetting): scen: Scenario = import_class(PROP_SETTING.scen)() @@ -62,4 +62,4 @@ def feedback(self, prev_out: dict[str, Any]): feedback = self.summarizer.generate_feedback(prev_out["running"], prev_out["propose"], self.trace) with logger.tag("ef"): # evaluate and feedback logger.log_object(feedback, tag="feedback") - self.trace.hist.append((prev_out["propose"],prev_out["running"] , feedback)) + self.trace.hist.append((prev_out["propose"], prev_out["running"], feedback)) diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py index 25506dcbe..c797d75f4 100644 --- a/rdagent/core/evolving_agent.py +++ b/rdagent/core/evolving_agent.py @@ -25,14 +25,16 @@ def multistep_evolve( evo: EvolvableSubjects, eva: Evaluator | Feedback, filter_final_evo: bool = False, - ) -> EvolvableSubjects: ... + ) -> EvolvableSubjects: + ... @abstractmethod def filter_evolvable_subjects_by_feedback( self, evo: EvolvableSubjects, feedback: Feedback | None, - ) -> EvolvableSubjects: ... + ) -> EvolvableSubjects: + ... class RAGEvoAgent(EvoAgent): @@ -52,7 +54,6 @@ def __init__( self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen - def multistep_evolve( self, evo: EvolvableSubjects, @@ -86,7 +87,9 @@ def multistep_evolve( es.feedback = ( # TODO: Due to the irregular design of rdagent.core.evaluation.Evaluator, # it fails mypy's test here, so we'll ignore this error for now. - eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge) # type: ignore[arg-type, call-arg] + eva + if isinstance(eva, Feedback) + else eva.evaluate(evo, queried_knowledge=queried_knowledge) # type: ignore[arg-type, call-arg] ) logger.log_object(es.feedback, tag="evolving feedback") diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index 1da6bd558..c1e598eda 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -99,12 +99,12 @@ def gen(self, trace: Trace) -> Hypothesis: # def gen(self, scenario_desc: str, ) -> Hypothesis: """ Motivation of the variable `scenario_desc`: - - Mocking a data-scientist is observing the scenario. + - Mocking a data-scientist is observing the scenario. - scenario_desc may conclude: - - data observation: - - Original or derivative - - Task information: + scenario_desc may include: + - data observation: + - Original or derivative + - Task information: """ diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index a01f2a640..2fdccc707 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -18,6 +18,7 @@ class SingletonBaseClass: Because we try to support defining Singleton with `class A(SingletonBaseClass)` instead of `A(metaclass=SingletonMeta)` this class becomes necessary. """ + _instance_dict: ClassVar[dict] = {} def __new__(cls, *args: Any, **kwargs: Any) -> Any: @@ -33,7 +34,6 @@ def __new__(cls, *args: Any, **kwargs: Any) -> Any: return cls._instance_dict[kwargs_hash] - def parse_json(response: str) -> Any: try: return json.loads(response) diff --git a/rdagent/log/base.py b/rdagent/log/base.py index bdcf3c048..305fb7d3e 100644 --- a/rdagent/log/base.py +++ b/rdagent/log/base.py @@ -2,19 +2,22 @@ from abc import abstractmethod from collections.abc import Generator +from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Literal, Optional, Union, Literal -from dataclasses import dataclass +from typing import Literal, Optional, Union @dataclass class Message: """The info unit of the storage""" + tag: str # namespace like like a.b.c level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] # The level of the logging timestamp: datetime # The time when the message is generated - caller: Optional[str] # The caller of the logging like `rdagent.oai.llm_utils:_create_chat_completion_inner_function:55`(file:func:line) + caller: Optional[ + str + ] # The caller of the logging like `rdagent.oai.llm_utils:_create_chat_completion_inner_function:55`(file:func:line) pid_trace: Optional[str] # The process id trace; A-B-C represents A create B, B create C content: object # The content @@ -39,7 +42,8 @@ class Storage: @abstractmethod def log( self, - obj: object, name: str = "", + obj: object, + name: str = "", save_type: Literal["json", "text", "pkl"] = "text", timestamp: datetime | None = None, **kwargs: dict, diff --git a/rdagent/log/logger.py b/rdagent/log/logger.py index fc987fead..87caf32c6 100644 --- a/rdagent/log/logger.py +++ b/rdagent/log/logger.py @@ -3,15 +3,17 @@ from contextlib import contextmanager from datetime import datetime, timezone from functools import partial +from logging import LogRecord from multiprocessing import Pipe from multiprocessing.connection import Connection from pathlib import Path -from typing import TYPE_CHECKING, Union, Generator, Dict, Any -from logging import LogRecord +from typing import TYPE_CHECKING, Any, Dict, Generator, Union from loguru import logger + if TYPE_CHECKING: from loguru import Record + from psutil import Process from rdagent.core.conf import RD_AGENT_SETTINGS diff --git a/rdagent/log/storage.py b/rdagent/log/storage.py index ff514892d..7ed014c52 100644 --- a/rdagent/log/storage.py +++ b/rdagent/log/storage.py @@ -1,9 +1,9 @@ -import re import json import pickle +import re from datetime import datetime, timezone from pathlib import Path -from typing import Literal, Generator, Union, Any, cast +from typing import Any, Generator, Literal, Union, cast from .base import Message, Storage @@ -17,7 +17,6 @@ class FileStorage(Storage): TODO: describe the storage format """ - def __init__(self, path: str | Path = "./log/") -> None: self.path = Path(path) self.path.mkdir(parents=True, exist_ok=True) @@ -69,7 +68,7 @@ def log( def iter_msg(self, watch: bool = False) -> Generator[Message, None, None]: msg_l = [] for file in self.path.glob("**/*.log"): - tag = '.'.join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3]) + tag = ".".join(str(file.relative_to(self.path)).replace("/", ".").split(".")[:-3]) pid = file.parent.name with file.open("r") as f: @@ -92,12 +91,7 @@ def iter_msg(self, watch: bool = False) -> Generator[Message, None, None]: message_content = content[message_start:message_end].strip() m = Message( - tag=tag, - level=level, - timestamp=timestamp, - caller=caller, - pid_trace=pid, - content=message_content + tag=tag, level=level, timestamp=timestamp, caller=caller, pid_trace=pid, content=message_content ) if isinstance(m.content, str) and "Logging object in" in m.content: @@ -119,7 +113,6 @@ def iter_msg(self, watch: bool = False) -> Generator[Message, None, None]: def truncate(self, time: datetime) -> None: # any message later than `time` will be removed for file in self.path.glob("**/*.log"): - with file.open("r") as f: content = f.read() @@ -135,7 +128,7 @@ def truncate(self, time: datetime) -> None: log_start = match.start() log_end = next_match.start() if next_match else len(content) - msg = content[match.end():log_end].strip() + msg = content[match.end() : log_end].strip() if timestamp > time: if "Logging object in" in msg: diff --git a/rdagent/log/ui/app.py b/rdagent/log/ui/app.py index 038dc8bfe..52060c681 100644 --- a/rdagent/log/ui/app.py +++ b/rdagent/log/ui/app.py @@ -1,9 +1,15 @@ -from rdagent.log.ui.web import WebView, SimpleTraceWindow, TraceObjWindow, mock_msg, TraceWindow -from rdagent.log.storage import FileStorage, Message -from rdagent.core.proposal import Trace -from pathlib import Path import pickle +from pathlib import Path +from rdagent.core.proposal import Trace +from rdagent.log.storage import FileStorage, Message +from rdagent.log.ui.web import ( + SimpleTraceWindow, + TraceObjWindow, + TraceWindow, + WebView, + mock_msg, +) # show logs folder WebView(TraceWindow()).display(FileStorage("/data/home/bowen/workspace/RD-Agent/log/yuante/2024-07-24_04-03-33-691119")) diff --git a/rdagent/log/ui/st_fixed_container.py b/rdagent/log/ui/st_fixed_container.py index ae3d8cc28..262f0388e 100644 --- a/rdagent/log/ui/st_fixed_container.py +++ b/rdagent/log/ui/st_fixed_container.py @@ -123,4 +123,4 @@ def st_fixed_container( st.container(border=True).write("This is a regular container.") for i in range(30): - st.write(f"Line {i}") \ No newline at end of file + st.write(f"Line {i}") diff --git a/rdagent/log/ui/web.py b/rdagent/log/ui/web.py index 16f68f8e7..83c66bffa 100644 --- a/rdagent/log/ui/web.py +++ b/rdagent/log/ui/web.py @@ -1,51 +1,45 @@ -import pandas as pd -import streamlit as st -import plotly.express as px import time - -from rdagent.log.base import Storage, View -from rdagent.log.base import Message -from datetime import timezone, datetime from collections import defaultdict from copy import deepcopy -from rdagent.core.proposal import Trace - +from datetime import datetime, timezone from typing import Callable, Type + +import pandas as pd +import plotly.express as px +import streamlit as st from streamlit.delta_generator import DeltaGenerator -from rdagent.core.proposal import Hypothesis, HypothesisFeedback +from rdagent.components.coder.factor_coder.CoSTEER.evaluators import ( + FactorSingleFeedback, +) +from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask +from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback +from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask +from rdagent.core.proposal import Hypothesis, HypothesisFeedback, Trace +from rdagent.log.base import Message, Storage, View from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment from rdagent.scenarios.qlib.experiment.model_experiment import QlibModelExperiment -from rdagent.components.coder.factor_coder.factor import FactorTask, FactorFBWorkspace -from rdagent.components.coder.factor_coder.CoSTEER.evaluators import FactorSingleFeedback -from rdagent.components.coder.model_coder.CoSTEER.evaluators import ModelCoderFeedback -from rdagent.components.coder.model_coder.model import ModelTask, ModelFBWorkspace - - st.set_page_config(layout="wide") TIME_DELAY = 0.001 -class WebView(View): - def __init__(self, ui: 'StWindow'): +class WebView(View): + def __init__(self, ui: "StWindow"): self.ui = ui # Save logs to your desired data structure # ... def display(self, s: Storage, watch: bool = False): - for msg in s.iter_msg(): # iterate overtime # NOTE: iter_msg will correctly seperate the information. # TODO: msg may support streaming mode. self.ui.consume_msg(msg) - class StWindow: - - def __init__(self, container: 'DeltaGenerator'): + def __init__(self, container: "DeltaGenerator"): self.container = container def consume_msg(self, msg: Message): @@ -54,31 +48,32 @@ def consume_msg(self, msg: Message): class LLMWindow(StWindow): - def __init__(self, container: 'DeltaGenerator', session_name: str="common"): + def __init__(self, container: "DeltaGenerator", session_name: str = "common"): self.session_name = session_name self.container = container.expander(f"{self.session_name} message") def consume_msg(self, msg: Message): - self.container.chat_message('user').markdown(f"{msg.content}") + self.container.chat_message("user").markdown(f"{msg.content}") class ProgressTabsWindow(StWindow): - ''' + """ For windows with stream messages, will refresh when a new tab is created. - ''' - def __init__(self, - container: 'DeltaGenerator', - inner_class: Type[StWindow] = StWindow, - mapper: Callable[[Message], str] = lambda x: x.pid_trace): - + """ + + def __init__( + self, + container: "DeltaGenerator", + inner_class: Type[StWindow] = StWindow, + mapper: Callable[[Message], str] = lambda x: x.pid_trace, + ): self.inner_class = inner_class self.mapper = mapper self.container = container.empty() self.tab_windows: dict[str, StWindow] = defaultdict(None) self.tab_caches: dict[str, list[Message]] = defaultdict(list) - - + def consume_msg(self, msg: Message): name = self.mapper(msg) @@ -93,61 +88,67 @@ def consume_msg(self, msg: Message): for id, name in enumerate(names): self.tab_windows[name] = self.inner_class(tabs[id]) - + # consume the cache for name in self.tab_caches: for msg in self.tab_caches[name]: self.tab_windows[name].consume_msg(msg) - + self.tab_caches[name].append(msg) self.tab_windows[name].consume_msg(msg) class ObjectsTabsWindow(StWindow): - def __init__(self, - container: 'DeltaGenerator', - inner_class: Type[StWindow] = StWindow, - mapper: Callable[[object], str] = lambda x: str(x), - tab_names: list[str] | None = None): + def __init__( + self, + container: "DeltaGenerator", + inner_class: Type[StWindow] = StWindow, + mapper: Callable[[object], str] = lambda x: str(x), + tab_names: list[str] | None = None, + ): self.inner_class = inner_class self.mapper = mapper self.container = container self.tab_names = tab_names - + def consume_msg(self, msg: Message): if isinstance(msg.content, list): if self.tab_names: - assert len(self.tab_names) == len(msg.content), "List of objects should have the same length as provided tab names." + assert len(self.tab_names) == len( + msg.content + ), "List of objects should have the same length as provided tab names." objs_dict = {self.tab_names[id]: obj for id, obj in enumerate(msg.content)} else: objs_dict = {self.mapper(obj): obj for obj in msg.content} elif not isinstance(msg.content, dict): raise ValueError("Message content should be a list or a dict of objects.") - + # two many tabs may cause display problem tab_names = list(objs_dict.keys()) tabs = [] for i in range(0, len(tab_names), 10): - tabs.extend(self.container.tabs(tab_names[i:i+10])) - + tabs.extend(self.container.tabs(tab_names[i : i + 10])) + for id, obj in enumerate(objs_dict.values()): - splited_msg = Message(tag=msg.tag, - level=msg.level, - timestamp=msg.timestamp, - caller=msg.caller, - pid_trace=msg.pid_trace, - content=obj) + splited_msg = Message( + tag=msg.tag, + level=msg.level, + timestamp=msg.timestamp, + caller=msg.caller, + pid_trace=msg.pid_trace, + content=obj, + ) self.inner_class(tabs[id]).consume_msg(splited_msg) class RoundTabsWindow(StWindow): - - def __init__(self, - container: 'DeltaGenerator', - new_tab_func: Callable[[Message], bool], - inner_class: Type[StWindow] = StWindow, - title: str = 'Round tabs'): - + def __init__( + self, + container: "DeltaGenerator", + new_tab_func: Callable[[Message], bool], + inner_class: Type[StWindow] = StWindow, + title: str = "Round tabs", + ): container.markdown(f"### **{title}**") self.inner_class = inner_class self.new_tab_func = new_tab_func @@ -156,42 +157,42 @@ def __init__(self, self.current_win = StWindow(container) self.tabs_c = container.empty() - def consume_msg(self, msg: Message): if self.new_tab_func(msg): self.round += 1 - self.current_win = self.inner_class(self.tabs_c.tabs([str(i) for i in range(1, self.round+1)])[-1]) + self.current_win = self.inner_class(self.tabs_c.tabs([str(i) for i in range(1, self.round + 1)])[-1]) self.current_win.consume_msg(msg) class HypothesisWindow(StWindow): - def consume_msg(self, msg: Message | Hypothesis): h: Hypothesis = msg.content if isinstance(msg, Message) else msg - self.container.markdown('#### **Hypothesis💡**') - self.container.markdown(f""" + self.container.markdown("#### **Hypothesis💡**") + self.container.markdown( + f""" - **Hypothesis**: {h.hypothesis} -- **Reason**: {h.reason}""") +- **Reason**: {h.reason}""" + ) class HypothesisFeedbackWindow(StWindow): - def consume_msg(self, msg: Message | HypothesisFeedback): h: HypothesisFeedback = msg.content if isinstance(msg, Message) else msg - self.container.markdown('#### **Hypothesis Feedback🔍**') - self.container.markdown(f""" + self.container.markdown("#### **Hypothesis Feedback🔍**") + self.container.markdown( + f""" - **Observations**: {h.observations} - **Hypothesis Evaluation**: {h.hypothesis_evaluation} - **New Hypothesis**: {h.new_hypothesis} - **Decision**: {h.decision} -- **Reason**: {h.reason}""") +- **Reason**: {h.reason}""" + ) class FactorTaskWindow(StWindow): - def consume_msg(self, msg: Message | FactorTask): ft: FactorTask = msg.content if isinstance(msg, Message) else msg @@ -199,14 +200,13 @@ def consume_msg(self, msg: Message | FactorTask): self.container.markdown(f"**Description**: {ft.factor_description}") self.container.latex(f"Formulation: {ft.factor_formulation}") - variables_df = pd.DataFrame(ft.variables, index=['Description']).T - variables_df.index.name = 'Variable' + variables_df = pd.DataFrame(ft.variables, index=["Description"]).T + variables_df.index.name = "Variable" self.container.table(variables_df) self.container.text(f"Factor resources: {ft.factor_resources}") class ModelTaskWindow(StWindow): - def consume_msg(self, msg: Message | ModelTask): mt: ModelTask = msg.content if isinstance(msg, Message) else msg @@ -214,18 +214,18 @@ def consume_msg(self, msg: Message | ModelTask): self.container.markdown(f"**Model Type**: {mt.model_type}") self.container.markdown(f"**Description**: {mt.description}") self.container.markdown(f"**Formulation**: {mt.formulation}") - - variables_df = pd.DataFrame(mt.variables, index=['Value']).T - variables_df.index.name = 'Variable' + + variables_df = pd.DataFrame(mt.variables, index=["Value"]).T + variables_df.index.name = "Variable" self.container.table(variables_df) class FactorFeedbackWindow(StWindow): - def consume_msg(self, msg: Message | FactorSingleFeedback): fb: FactorSingleFeedback = msg.content if isinstance(msg, Message) else msg - self.container.markdown(f"""### :blue[Factor Execution Feedback] + self.container.markdown( + f"""### :blue[Factor Execution Feedback] {fb.execution_feedback} ### :blue[Factor Code Feedback] {fb.code_feedback} @@ -235,15 +235,16 @@ def consume_msg(self, msg: Message | FactorSingleFeedback): {fb.final_feedback} ### :blue[Factor Final Decision] This implementation is {'SUCCESS' if fb.final_decision else 'FAIL'}. -""") +""" + ) class ModelFeedbackWindow(StWindow): - def consume_msg(self, msg: Message | ModelCoderFeedback): mb: ModelCoderFeedback = msg.content if isinstance(msg, Message) else msg - self.container.markdown(f"""### :blue[Model Execution Feedback] + self.container.markdown( + f"""### :blue[Model Execution Feedback] {mb.execution_feedback} ### :blue[Model Shape Feedback] {mb.shape_feedback} @@ -255,11 +256,12 @@ def consume_msg(self, msg: Message | ModelCoderFeedback): {mb.final_feedback} ### :blue[Model Final Decision] This implementation is {'SUCCESS' if mb.final_decision else 'FAIL'}. -""") +""" + ) class WorkspaceWindow(StWindow): - def __init__(self, container: 'DeltaGenerator', show_task_info: bool = False): + def __init__(self, container: "DeltaGenerator", show_task_info: bool = False): self.container = container self.show_task_info = show_task_info @@ -267,21 +269,22 @@ def consume_msg(self, msg: Message | FactorFBWorkspace | ModelFBWorkspace): ws: FactorFBWorkspace | ModelFBWorkspace = msg.content if isinstance(msg, Message) else msg # no workspace - if ws is None: return + if ws is None: + return # task info if self.show_task_info: task_msg = deepcopy(msg) task_msg.content = ws.target_task if isinstance(ws, FactorFBWorkspace): - self.container.subheader('Factor Info') + self.container.subheader("Factor Info") FactorTaskWindow(self.container.container()).consume_msg(task_msg) else: - self.container.subheader('Model Info') + self.container.subheader("Model Info") ModelTaskWindow(self.container.container()).consume_msg(task_msg) # task codes - for k,v in ws.code_dict.items(): + for k, v in ws.code_dict.items(): self.container.markdown(f"`{k}`") self.container.code(v, language="python") @@ -302,24 +305,25 @@ def consume_msg(self, msg: Message | QlibFactorExperiment): if self.show_task_info: ftm_msg = deepcopy(msg) ftm_msg.content = [ws for ws in exp.sub_workspace_list if ws] - self.container.markdown('**Factor Tasks**') - ObjectsTabsWindow(self.container.container(), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.factor_name, - ).consume_msg(ftm_msg) + self.container.markdown("**Factor Tasks**") + ObjectsTabsWindow( + self.container.container(), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.factor_name, + ).consume_msg(ftm_msg) # result - self.container.markdown('**Results**') - results = pd.DataFrame({f'base_exp_{id}':e.result for id, e in enumerate(exp.based_experiments)}) - results['now'] = exp.result + self.container.markdown("**Results**") + results = pd.DataFrame({f"base_exp_{id}": e.result for id, e in enumerate(exp.based_experiments)}) + results["now"] = exp.result - self.container.expander('results table').table(results) + self.container.expander("results table").table(results) try: - bar_chart = px.bar(results, orientation='h', barmode='group') - self.container.expander('results chart').plotly_chart(bar_chart) + bar_chart = px.bar(results, orientation="h", barmode="group") + self.container.expander("results chart").plotly_chart(bar_chart) except: - self.container.text('Results are incomplete.') + self.container.text("Results are incomplete.") class QlibModelExpWindow(StWindow): @@ -334,44 +338,45 @@ def consume_msg(self, msg: Message | QlibModelExperiment): if self.show_task_info: _msg = deepcopy(msg) _msg.content = [ws for ws in exp.sub_workspace_list if ws] - self.container.markdown('**Model Tasks**') - ObjectsTabsWindow(self.container.container(), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.name, - ).consume_msg(_msg) + self.container.markdown("**Model Tasks**") + ObjectsTabsWindow( + self.container.container(), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.name, + ).consume_msg(_msg) # result - self.container.subheader('Results', divider=True) - results = pd.DataFrame({f'base_exp_{id}':e.result for id, e in enumerate(exp.based_experiments)}) - results['now'] = exp.result + self.container.subheader("Results", divider=True) + results = pd.DataFrame({f"base_exp_{id}": e.result for id, e in enumerate(exp.based_experiments)}) + results["now"] = exp.result - self.container.expander('results table').table(results) + self.container.expander("results table").table(results) class SimpleTraceWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator' = st.container(), show_llm: bool = False, show_common_logs: bool = False): + def __init__( + self, container: "DeltaGenerator" = st.container(), show_llm: bool = False, show_common_logs: bool = False + ): super().__init__(container) self.show_llm = show_llm self.show_common_logs = show_common_logs - self.pid_trace = '' - self.current_tag = '' + self.pid_trace = "" + self.current_tag = "" self.current_win = StWindow(self.container) self.evolving_tasks: list[str] = [] def consume_msg(self, msg: Message): - # divide tag levels if len(msg.tag) > len(self.current_tag): # write a header about current task, if it is llm message, not write. - if not msg.tag.endswith('llm_messages'): - self.container.header(msg.tag.replace('.', ' ➡ '), divider=True) - + if not msg.tag.endswith("llm_messages"): + self.container.header(msg.tag.replace(".", " ➡ "), divider=True) + self.current_tag = msg.tag # set log writer (window) according to msg - if msg.tag.endswith('llm_messages'): + if msg.tag.endswith("llm_messages"): # llm messages logs if not self.show_llm: return @@ -392,29 +397,41 @@ def consume_msg(self, msg: Message): if len(msg.content) == 0: return if isinstance(msg.content[0], FactorTask): - self.current_win = ObjectsTabsWindow(self.container.expander('Factor Tasks'), FactorTaskWindow, lambda x: x.factor_name) + self.current_win = ObjectsTabsWindow( + self.container.expander("Factor Tasks"), FactorTaskWindow, lambda x: x.factor_name + ) elif isinstance(msg.content[0], ModelTask): - self.current_win = ObjectsTabsWindow(self.container.expander('Model Tasks'), ModelTaskWindow, lambda x: x.name) - + self.current_win = ObjectsTabsWindow( + self.container.expander("Model Tasks"), ModelTaskWindow, lambda x: x.name + ) + elif isinstance(msg.content[0], FactorFBWorkspace): - self.current_win = ObjectsTabsWindow(self.container.expander('Factor Workspaces'), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.factor_name) + self.current_win = ObjectsTabsWindow( + self.container.expander("Factor Workspaces"), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.factor_name, + ) self.evolving_tasks = [m.target_task.factor_name for m in msg.content] elif isinstance(msg.content[0], ModelFBWorkspace): - self.current_win = ObjectsTabsWindow(self.container.expander('Model Workspaces'), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.name) + self.current_win = ObjectsTabsWindow( + self.container.expander("Model Workspaces"), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.name, + ) self.evolving_tasks = [m.target_task.name for m in msg.content] elif isinstance(msg.content[0], FactorSingleFeedback): - self.current_win = ObjectsTabsWindow(self.container.expander('Factor Feedbacks'), - inner_class=FactorFeedbackWindow, - tab_names=self.evolving_tasks) + self.current_win = ObjectsTabsWindow( + self.container.expander("Factor Feedbacks"), + inner_class=FactorFeedbackWindow, + tab_names=self.evolving_tasks, + ) elif isinstance(msg.content[0], ModelCoderFeedback): - self.current_win = ObjectsTabsWindow(self.container.expander('Model Feedbacks'), - inner_class=ModelFeedbackWindow, - tab_names=self.evolving_tasks) + self.current_win = ObjectsTabsWindow( + self.container.expander("Model Feedbacks"), + inner_class=ModelFeedbackWindow, + tab_names=self.evolving_tasks, + ) else: # common logs if not self.show_common_logs: @@ -425,12 +442,11 @@ def consume_msg(self, msg: Message): def mock_msg(obj) -> Message: - return Message(tag='mock', level='INFO', timestamp=datetime.now(), pid_trace='000', caller='mock',content=obj) + return Message(tag="mock", level="INFO", timestamp=datetime.now(), pid_trace="000", caller="mock", content=obj) class TraceObjWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator' = st.container()): + def __init__(self, container: "DeltaGenerator" = st.container()): self.container = container def consume_msg(self, msg: Message | Trace): @@ -440,7 +456,7 @@ def consume_msg(self, msg: Message | Trace): trace = msg for id, (h, e, hf) in enumerate(trace.hist): - self.container.header(f'Trace History {id}', divider=True) + self.container.header(f"Trace History {id}", divider=True) HypothesisWindow(self.container).consume_msg(mock_msg(h)) if isinstance(e, QlibFactorExperiment): QlibFactorExpWindow(self.container).consume_msg(mock_msg(e)) @@ -450,70 +466,74 @@ def consume_msg(self, msg: Message | Trace): class ResearchWindow(StWindow): - def consume_msg(self, msg: Message): - if msg.tag.endswith('hypothesis generation'): + if msg.tag.endswith("hypothesis generation"): HypothesisWindow(self.container.container()).consume_msg(msg) - elif msg.tag.endswith('experiment generation'): + elif msg.tag.endswith("experiment generation"): if isinstance(msg.content, list): if isinstance(msg.content[0], FactorTask): - self.container.markdown('**Factor Tasks**') - ObjectsTabsWindow(self.container.container(), FactorTaskWindow, lambda x: x.factor_name).consume_msg(msg) + self.container.markdown("**Factor Tasks**") + ObjectsTabsWindow( + self.container.container(), FactorTaskWindow, lambda x: x.factor_name + ).consume_msg(msg) elif isinstance(msg.content[0], ModelTask): - self.container.markdown('**Model Tasks**') + self.container.markdown("**Model Tasks**") ObjectsTabsWindow(self.container.container(), ModelTaskWindow, lambda x: x.name).consume_msg(msg) class EvolvingWindow(StWindow): - def __init__(self, container: 'DeltaGenerator'): + def __init__(self, container: "DeltaGenerator"): self.container = container self.evolving_tasks: list[str] = [] def consume_msg(self, msg: Message): - if msg.tag.endswith('evolving code'): + if msg.tag.endswith("evolving code"): if isinstance(msg.content, list): msg.content = [m for m in msg.content if m] if len(msg.content) == 0: return if isinstance(msg.content[0], FactorFBWorkspace): - self.container.markdown('**Factor Codes**') - ObjectsTabsWindow(self.container.container(), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.factor_name).consume_msg(msg) + self.container.markdown("**Factor Codes**") + ObjectsTabsWindow( + self.container.container(), + inner_class=WorkspaceWindow, + mapper=lambda x: x.target_task.factor_name, + ).consume_msg(msg) self.evolving_tasks = [m.target_task.factor_name for m in msg.content] elif isinstance(msg.content[0], ModelFBWorkspace): - self.container.markdown('**Model Codes**') - ObjectsTabsWindow(self.container.container(), - inner_class=WorkspaceWindow, - mapper=lambda x: x.target_task.name).consume_msg(msg) + self.container.markdown("**Model Codes**") + ObjectsTabsWindow( + self.container.container(), inner_class=WorkspaceWindow, mapper=lambda x: x.target_task.name + ).consume_msg(msg) self.evolving_tasks = [m.target_task.name for m in msg.content] - elif msg.tag.endswith('evolving feedback'): + elif msg.tag.endswith("evolving feedback"): if isinstance(msg.content, list): msg.content = [m for m in msg.content if m] if len(msg.content) == 0: return if isinstance(msg.content[0], FactorSingleFeedback): - self.container.markdown('**Factor Feedbacks🔍**') - ObjectsTabsWindow(self.container.container(), - inner_class=FactorFeedbackWindow, - tab_names=self.evolving_tasks).consume_msg(msg) + self.container.markdown("**Factor Feedbacks🔍**") + ObjectsTabsWindow( + self.container.container(), inner_class=FactorFeedbackWindow, tab_names=self.evolving_tasks + ).consume_msg(msg) elif isinstance(msg.content[0], ModelCoderFeedback): - self.container.markdown('**Model Feedbacks🔍**') - ObjectsTabsWindow(self.container.container(), - inner_class=ModelFeedbackWindow, - tab_names=self.evolving_tasks).consume_msg(msg) + self.container.markdown("**Model Feedbacks🔍**") + ObjectsTabsWindow( + self.container.container(), inner_class=ModelFeedbackWindow, tab_names=self.evolving_tasks + ).consume_msg(msg) class DevelopmentWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator'): - self.E_win = RoundTabsWindow(container.container(), - new_tab_func=lambda x: x.tag.endswith('evolving code'), - inner_class=EvolvingWindow, - title='Evolving Loops🔧') + def __init__(self, container: "DeltaGenerator"): + self.E_win = RoundTabsWindow( + container.container(), + new_tab_func=lambda x: x.tag.endswith("evolving code"), + inner_class=EvolvingWindow, + title="Evolving Loops🔧", + ) def consume_msg(self, msg: Message): - if 'evolving' in msg.tag: + if "evolving" in msg.tag: self.E_win.consume_msg(msg) # elif msg.tag.endswith('result'): # self.container.subheader('Results') @@ -528,8 +548,7 @@ def consume_msg(self, msg: Message): class FeedbackWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator'): + def __init__(self, container: "DeltaGenerator"): self.container = container def consume_msg(self, msg: Message): @@ -542,8 +561,7 @@ def consume_msg(self, msg: Message): class SingleRDLoopWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator'): + def __init__(self, container: "DeltaGenerator"): self.container = container col1, col2 = self.container.columns([2, 3]) self.R_win = ResearchWindow(col1.container(border=True)) @@ -551,58 +569,66 @@ def __init__(self, container: 'DeltaGenerator'): self.D_win = DevelopmentWindow(col2.container(border=True)) def consume_msg(self, msg: Message): - tags = msg.tag.split('.') - if 'r' in tags: + tags = msg.tag.split(".") + if "r" in tags: self.R_win.consume_msg(msg) - elif 'd' in tags: + elif "d" in tags: self.D_win.consume_msg(msg) - elif 'ef' in tags: + elif "ef" in tags: self.F_win.consume_msg(msg) class TraceWindow(StWindow): - - def __init__(self, container: 'DeltaGenerator' = st.container(), show_llm: bool = False, show_common_logs: bool = False): + def __init__( + self, container: "DeltaGenerator" = st.container(), show_llm: bool = False, show_common_logs: bool = False + ): self.show_llm = show_llm self.show_common_logs = show_common_logs top_container = container.container() - col1, col2 = top_container.columns([2,3]) + col1, col2 = top_container.columns([2, 3]) chart_c = col2.container(border=True, height=300) - chart_c.markdown('**Metrics📈**') + chart_c.markdown("**Metrics📈**") self.chart_c = chart_c.empty() hypothesis_status_c = col1.container(border=True, height=300) - hypothesis_status_c.markdown('**Hypotheses🏅**') + hypothesis_status_c.markdown("**Hypotheses🏅**") self.summary_c = hypothesis_status_c.empty() - self.RDL_win = RoundTabsWindow(container.container(), - new_tab_func=lambda x: x.tag.endswith('hypothesis generation'), - inner_class=SingleRDLoopWindow, - title='R&D Loops♾️') - + self.RDL_win = RoundTabsWindow( + container.container(), + new_tab_func=lambda x: x.tag.endswith("hypothesis generation"), + inner_class=SingleRDLoopWindow, + title="R&D Loops♾️", + ) + self.hypothesis_decisions = defaultdict(bool) self.current_hypothesis = None self.results = [] def consume_msg(self, msg: Message): - if not self.show_llm and 'llm_messages' in msg.tag: + if not self.show_llm and "llm_messages" in msg.tag: return if not self.show_common_logs and isinstance(msg.content, str): return if isinstance(msg.content, dict): return - if msg.tag.endswith('hypothesis generation'): + if msg.tag.endswith("hypothesis generation"): self.current_hypothesis = msg.content.hypothesis - elif msg.tag.endswith('ef.feedback'): + elif msg.tag.endswith("ef.feedback"): self.hypothesis_decisions[self.current_hypothesis] = msg.content.decision - self.summary_c.markdown('\n'.join(f"{id+1}. :green[{h}]\n" if d else f"{id+1}. {h}\n" for id,(h,d) in enumerate(self.hypothesis_decisions.items()))) - elif msg.tag.endswith('ef.model runner result') or msg.tag.endswith('ef.factor runner result'): + self.summary_c.markdown( + "\n".join( + f"{id+1}. :green[{h}]\n" if d else f"{id+1}. {h}\n" + for id, (h, d) in enumerate(self.hypothesis_decisions.items()) + ) + ) + elif msg.tag.endswith("ef.model runner result") or msg.tag.endswith("ef.factor runner result"): self.results.append(msg.content.result) if len(self.results) == 1: self.chart_c.table(self.results[0]) else: - df = pd.DataFrame(self.results, index=range(1, len(self.results)+1)) + df = pd.DataFrame(self.results, index=range(1, len(self.results) + 1)) fig = px.line(df, x=df.index, y=df.columns, markers=True) self.chart_c.plotly_chart(fig) diff --git a/rdagent/log/utils.py b/rdagent/log/utils.py index 746db5201..b87682bce 100644 --- a/rdagent/log/utils.py +++ b/rdagent/log/utils.py @@ -1,7 +1,6 @@ import inspect import re - -from typing import Union, Dict, TypedDict, Optional +from typing import Dict, Optional, TypedDict, Union class LogColors: diff --git a/rdagent/scenarios/data_mining/developer/feedback.py b/rdagent/scenarios/data_mining/developer/feedback.py index 3f30c0fc4..af18f0c4e 100644 --- a/rdagent/scenarios/data_mining/developer/feedback.py +++ b/rdagent/scenarios/data_mining/developer/feedback.py @@ -18,7 +18,7 @@ from rdagent.oai.llm_utils import APIBackend from rdagent.utils import convert2bool -feedback_prompts = Prompts(file_path=Path(__file__).parent.parent.parent/ "qlib" / "prompts.yaml") +feedback_prompts = Prompts(file_path=Path(__file__).parent.parent.parent / "qlib" / "prompts.yaml") DIRNAME = Path(__file__).absolute().resolve().parent diff --git a/rdagent/scenarios/data_mining/developer/model_runner.py b/rdagent/scenarios/data_mining/developer/model_runner.py index 8b643bf93..a3ef7e3d8 100644 --- a/rdagent/scenarios/data_mining/developer/model_runner.py +++ b/rdagent/scenarios/data_mining/developer/model_runner.py @@ -15,7 +15,6 @@ class DMModelRunner(CachedRunner[DMModelExperiment]): - def develop(self, exp: DMModelExperiment) -> DMModelExperiment: if RUNNER_SETTINGS.cache_result: cache_hit, result = self.get_cache_result(exp) diff --git a/rdagent/scenarios/data_mining/experiment/model_experiment.py b/rdagent/scenarios/data_mining/experiment/model_experiment.py index f737f83da..216524cf9 100644 --- a/rdagent/scenarios/data_mining/experiment/model_experiment.py +++ b/rdagent/scenarios/data_mining/experiment/model_experiment.py @@ -22,11 +22,11 @@ class DMModelScenario(Scenario): @property def background(self) -> str: return prompt_dict["dm_model_background"] - + @property def source_data(self) -> str: raise NotImplementedError("source_data is not implemented") - + @property def output_format(self) -> str: return prompt_dict["dm_model_output_format"] @@ -38,9 +38,9 @@ def interface(self) -> str: @property def simulator(self) -> str: return prompt_dict["dm_model_simulator"] - + @property - def rich_style_description(self)->str: + def rich_style_description(self) -> str: return "Below is MIMIC Model Evolving Automatic R&D Demo." def get_scenario_all_desc(self) -> str: diff --git a/rdagent/scenarios/data_mining/experiment/model_template/train.py b/rdagent/scenarios/data_mining/experiment/model_template/train.py index 74440e1c4..05a670c9a 100644 --- a/rdagent/scenarios/data_mining/experiment/model_template/train.py +++ b/rdagent/scenarios/data_mining/experiment/model_template/train.py @@ -1,21 +1,23 @@ +import os +import random from pathlib import Path + +import numpy as np import pandas as pd +import sparse import torch -import torch.nn.functional as F -from torchvision import transforms, datasets -from torch.utils.data import DataLoader, Dataset import torch.nn as nn -import sparse -import random -import os +import torch.nn.functional as F from model import model_cls from sklearn.metrics import accuracy_score, roc_auc_score -import numpy as np +from torch.utils.data import DataLoader, Dataset +from torchvision import datasets, transforms # Set device for training device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # device = torch.device("cpu") + class MyDataset(Dataset): def __init__(self, x, label, device): self.x1 = x @@ -28,9 +30,11 @@ def __len__(self): def __getitem__(self, idx): if torch.is_tensor(idx): idx = idx.tolist() - return torch.FloatTensor(self.x1[idx]).to(self.device), \ - torch.tensor(self.label[idx], dtype=torch.float).to(self.device) - + return torch.FloatTensor(self.x1[idx]).to(self.device), torch.tensor(self.label[idx], dtype=torch.float).to( + self.device + ) + + def collate_fn(batch): x, label = [], [] for data in batch: @@ -39,12 +43,12 @@ def collate_fn(batch): return torch.stack(x, 0), torch.stack(label, 0) -datapath = '/root/.data' +datapath = "/root/.data" # datapath = '/home/v-suhancui/RD-Agent/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3' -X = sparse.load_npz(datapath+'/features/ARF_12h/X.npz').todense() -df_pop = pd.read_csv(datapath+'/population/ARF_12h.csv')['ARF_LABEL'] +X = sparse.load_npz(datapath + "/features/ARF_12h/X.npz").todense() +df_pop = pd.read_csv(datapath + "/population/ARF_12h.csv")["ARF_LABEL"] X = X.transpose(0, 2, 1) @@ -56,8 +60,12 @@ def collate_fn(batch): X_test, y_test = X[indices[split_point:]], np.array(df_pop[indices[split_point:]]) -train_dataloader = DataLoader(MyDataset(X_train, y_train, device), collate_fn=collate_fn, shuffle=True, drop_last=True, batch_size=64) -test_dataloader = DataLoader(MyDataset(X_test, y_test, device), collate_fn=collate_fn, shuffle=False, drop_last=False, batch_size=64) +train_dataloader = DataLoader( + MyDataset(X_train, y_train, device), collate_fn=collate_fn, shuffle=True, drop_last=True, batch_size=64 +) +test_dataloader = DataLoader( + MyDataset(X_test, y_test, device), collate_fn=collate_fn, shuffle=False, drop_last=False, batch_size=64 +) num_features = 4816 num_timesteps = 12 @@ -88,5 +96,5 @@ def collate_fn(batch): print(acc) # Save the predictions to submission.csv -with open('./submission.txt', 'w') as f: - f.write(str(acc)) \ No newline at end of file +with open("./submission.txt", "w") as f: + f.write(str(acc)) diff --git a/rdagent/scenarios/data_mining/experiment/workspace.py b/rdagent/scenarios/data_mining/experiment/workspace.py index f70fd090b..e2e9d3a16 100644 --- a/rdagent/scenarios/data_mining/experiment/workspace.py +++ b/rdagent/scenarios/data_mining/experiment/workspace.py @@ -2,10 +2,11 @@ import pandas as pd +from rdagent.app.data_mining.conf import PROP_SETTING from rdagent.core.experiment import FBWorkspace from rdagent.log import rdagent_logger as logger from rdagent.utils.env import DMDockerEnv -from rdagent.app.data_mining.conf import PROP_SETTING + class DMFBWorkspace(FBWorkspace): def __init__(self, template_folder_path: Path, *args, **kwargs) -> None: @@ -27,5 +28,5 @@ def execute(self, run_env: dict = {}, *args, **kwargs) -> str: if not csv_path.exists(): logger.error(f"File {csv_path} does not exist.") return None - with open(self.workspace_path / "submission.txt", 'r') as f: + with open(self.workspace_path / "submission.txt", "r") as f: return f.read() diff --git a/rdagent/scenarios/data_mining/proposal/model_proposal.py b/rdagent/scenarios/data_mining/proposal/model_proposal.py index 32ec88962..064a1a9dd 100644 --- a/rdagent/scenarios/data_mining/proposal/model_proposal.py +++ b/rdagent/scenarios/data_mining/proposal/model_proposal.py @@ -30,6 +30,7 @@ class DMModelHypothesisGen(ModelHypothesisGen): class XXXDMModelHypothesisGen(DMModelHypothesisGen): prompts: Prompts = a_specifc_prompt_dict """ + def __init__(self, scen: Scenario) -> Tuple[dict, bool]: super().__init__(scen) @@ -43,7 +44,7 @@ def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: "hypothesis_and_feedback": hypothesis_feedback, "RAG": "", "hypothesis_output_format": prompt_dict["hypothesis_output_format"], - "hypothesis_specification": prompt_dict["model_hypothesis_specification"] + "hypothesis_specification": prompt_dict["model_hypothesis_specification"], } return context_dict, True @@ -89,7 +90,9 @@ def convert_response(self, response: str, trace: Trace) -> ModelExperiment: variables = response_dict[model_name]["variables"] hyperparameters = response_dict[model_name]["hyperparameters"] model_type = response_dict[model_name]["model_type"] - tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)) + tasks.append( + ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type) + ) exp = DMModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] return exp diff --git a/rdagent/scenarios/qlib/developer/feedback.py b/rdagent/scenarios/qlib/developer/feedback.py index 612e09f1b..7e62071f9 100644 --- a/rdagent/scenarios/qlib/developer/feedback.py +++ b/rdagent/scenarios/qlib/developer/feedback.py @@ -1,8 +1,8 @@ import json from pathlib import Path -from jinja2 import Environment, StrictUndefined import pandas as pd +from jinja2 import Environment, StrictUndefined from rdagent.core.experiment import Experiment from rdagent.core.prompts import Prompts @@ -19,37 +19,38 @@ feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") DIRNAME = Path(__file__).absolute().resolve().parent + def process_results(current_result, sota_result): # Convert the results to dataframes current_df = pd.DataFrame(current_result) sota_df = pd.DataFrame(sota_result) - + # Set the metric as the index - current_df.index.name = 'metric' - sota_df.index.name = 'metric' - + current_df.index.name = "metric" + sota_df.index.name = "metric" + # Rename the value column to reflect the result type - current_df.rename(columns={'0': 'Current Result'}, inplace=True) - sota_df.rename(columns={'0': 'SOTA Result'}, inplace=True) - + current_df.rename(columns={"0": "Current Result"}, inplace=True) + sota_df.rename(columns={"0": "SOTA Result"}, inplace=True) + # Combine the dataframes on the Metric index combined_df = pd.concat([current_df, sota_df], axis=1) - + # Select important metrics for comparison important_metrics = [ - 'Rank ICIR', - '1day.excess_return_without_cost.max_drawdown', - '1day.excess_return_without_cost.information_ratio', - '1day.excess_return_without_cost.annualized_return', - '1day.excess_return_with_cost.max_drawdown', - '1day.excess_return_with_cost.information_ratio', - '1day.excess_return_with_cost.annualized_return', - 'IC', + "Rank ICIR", + "1day.excess_return_without_cost.max_drawdown", + "1day.excess_return_without_cost.information_ratio", + "1day.excess_return_without_cost.annualized_return", + "1day.excess_return_with_cost.max_drawdown", + "1day.excess_return_with_cost.information_ratio", + "1day.excess_return_with_cost.annualized_return", + "IC", ] - + # Filter the combined DataFrame to retain only the important metrics filtered_combined_df = combined_df.loc[important_metrics] - + return filtered_combined_df.to_dict() diff --git a/rdagent/scenarios/qlib/experiment/factor_experiment.py b/rdagent/scenarios/qlib/experiment/factor_experiment.py index 09e166d58..f245cffe5 100644 --- a/rdagent/scenarios/qlib/experiment/factor_experiment.py +++ b/rdagent/scenarios/qlib/experiment/factor_experiment.py @@ -39,9 +39,9 @@ def interface(self) -> str: @property def simulator(self) -> str: return prompt_dict["qlib_factor_simulator"] - + @property - def rich_style_description(self)->str: + def rich_style_description(self) -> str: return "Below is QlibFactor Evolving Automatic R&D Demo." def get_scenario_all_desc(self) -> str: diff --git a/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py index d08e3b4ce..928cc6451 100644 --- a/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py +++ b/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py @@ -45,4 +45,4 @@ print(f"Output has been saved to {output_path}") ret_data_frame = latest_recorder.load_object("portfolio_analysis/report_normal_1day.pkl") - ret_data_frame.to_pickle("ret.pkl") \ No newline at end of file + ret_data_frame.to_pickle("ret.pkl") diff --git a/rdagent/scenarios/qlib/experiment/model_experiment.py b/rdagent/scenarios/qlib/experiment/model_experiment.py index 3167c4f02..f1c99c8d2 100644 --- a/rdagent/scenarios/qlib/experiment/model_experiment.py +++ b/rdagent/scenarios/qlib/experiment/model_experiment.py @@ -38,10 +38,10 @@ def interface(self) -> str: @property def simulator(self) -> str: return prompt_dict["qlib_model_simulator"] - + @property - def rich_style_description(self)->str: - return ''' + def rich_style_description(self) -> str: + return """ ### Qlib Model Evolving Automatic R&D Demo #### Overview @@ -75,7 +75,7 @@ def rich_style_description(self)->str: To demonstrate the dynamic evolution of models through the Qlib platform, emphasizing how each iteration enhances the accuracy and reliability of the resulting models. - ''' + """ def get_scenario_all_desc(self) -> str: return f"""Background of the scenario: diff --git a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py index 01976cb92..928cc6451 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py +++ b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py @@ -39,7 +39,6 @@ # Load the specified file from the latest recorder metrics = pd.Series(latest_recorder.list_metrics()) - output_path = Path(__file__).resolve().parent / "qlib_res.csv" metrics.to_csv(output_path) diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py index 28c37b108..5f080f80e 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py @@ -56,9 +56,7 @@ def load(self, json_file_path: Path) -> list: variables=factor_data["variables"], ) gt = FactorFBWorkspace(task) - code = { - "factor.py": factor_data["gt_code"] - } + code = {"factor.py": factor_data["gt_code"]} gt.inject_code(**code) gt.execute() TestData.target_task.sub_tasks.append(task) diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py index d3d4f4718..b0c5e9ee6 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py @@ -196,7 +196,11 @@ def __extract_factor_and_formulation_from_one_report( factor_dict, ) for factor_name in factor_dict: - if factor_name not in factor_to_formulation or "formulation" not in factor_to_formulation[factor_name] or "variables" not in factor_to_formulation[factor_name]: + if ( + factor_name not in factor_to_formulation + or "formulation" not in factor_to_formulation[factor_name] + or "variables" not in factor_to_formulation[factor_name] + ): continue final_factor_dict_to_one_report.setdefault(factor_name, {}) @@ -272,7 +276,6 @@ def merge_file_to_factor_dict_to_factor_dict( def __check_factor_dict_viability_simulate_json_mode( factor_df_string: str, ) -> dict[str, dict[str, str]]: - extract_result_resp = APIBackend().build_messages_and_create_chat_completion( system_prompt=document_process_prompts["factor_viability_system"], user_prompt=factor_df_string, @@ -512,11 +515,11 @@ def load(self, file_or_folder_path: Path) -> dict: logger.log_object(docs_dict) selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1) - + with logger.tag("file_to_factor_result"): file_to_factor_result = extract_factors_from_report_dict(docs_dict, selected_report_dict) logger.log_object(file_to_factor_result) - + with logger.tag("factor_dict"): factor_dict = merge_file_to_factor_dict_to_factor_dict(file_to_factor_result) logger.log_object(factor_dict) @@ -526,5 +529,5 @@ def load(self, file_or_folder_path: Path) -> dict: logger.log_object(filtered_factor_dict) # factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability) - + return FactorExperimentLoaderFromDict().load(filtered_factor_dict) diff --git a/rdagent/scenarios/qlib/proposal/model_proposal.py b/rdagent/scenarios/qlib/proposal/model_proposal.py index fb2bcefa5..796e0a1ca 100644 --- a/rdagent/scenarios/qlib/proposal/model_proposal.py +++ b/rdagent/scenarios/qlib/proposal/model_proposal.py @@ -33,13 +33,17 @@ def prepare_context(self, trace: Trace) -> Tuple[dict, bool]: "hypothesis_and_feedback": hypothesis_feedback, "RAG": "In Quantitative Finance, market data could be time-series, and GRU model/LSTM model are suitable for them. Do not generate GNN model as for now.", "hypothesis_output_format": prompt_dict["hypothesis_output_format"], - "hypothesis_specification": prompt_dict["model_hypothesis_specification"] + "hypothesis_specification": prompt_dict["model_hypothesis_specification"], } return context_dict, True def convert_response(self, response: str) -> ModelHypothesis: response_dict = json.loads(response) - hypothesis = QlibModelHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"], concise_reason=response_dict["concise_reason"]) + hypothesis = QlibModelHypothesis( + hypothesis=response_dict["hypothesis"], + reason=response_dict["reason"], + concise_reason=response_dict["concise_reason"], + ) return hypothesis @@ -79,7 +83,9 @@ def convert_response(self, response: str, trace: Trace) -> ModelExperiment: variables = response_dict[model_name]["variables"] hyperparameters = response_dict[model_name]["hyperparameters"] model_type = response_dict[model_name]["model_type"] - tasks.append(ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type)) + tasks.append( + ModelTask(model_name, description, formulation, architecture, variables, hyperparameters, model_type) + ) exp = QlibModelExperiment(tasks) exp.based_experiments = [t[1] for t in trace.hist if t[2]] return exp diff --git a/rdagent/utils/agent/ret.py b/rdagent/utils/agent/ret.py index 8c8c29f40..6aad34334 100644 --- a/rdagent/utils/agent/ret.py +++ b/rdagent/utils/agent/ret.py @@ -3,8 +3,8 @@ We think this part can be shared. """ -from abc import abstractclassmethod import re +from abc import abstractclassmethod from typing import Any from rdagent.utils.agent.tpl import T diff --git a/rdagent/utils/agent/tpl.py b/rdagent/utils/agent/tpl.py index e7ec2c567..a3047b9f2 100644 --- a/rdagent/utils/agent/tpl.py +++ b/rdagent/utils/agent/tpl.py @@ -4,13 +4,12 @@ The motivation of tempalte and AgentOutput Design """ -from typing import Any -from jinja2 import Environment, StrictUndefined - +import inspect from pathlib import Path +from typing import Any import yaml -import inspect +from jinja2 import Environment, StrictUndefined from rdagent.core.utils import SingletonBaseClass @@ -21,9 +20,10 @@ # class T(SingletonBaseClass): TODO: singleton does not support args now. class T: """Use the simplest way to (C)reate a Template and (r)ender it!!""" + def __init__(self, uri: str): """ - here are some uri usages + here are some uri usages case 1) "a.b.c:x.y.z" It will load DIRNAME/a/b/c.yaml as `yaml` and load yaml[x][y][z] case 2) ".c:x.y.z" @@ -38,16 +38,16 @@ def __init__(self, uri: str): caller_dir = Path(caller_module.__file__).parent # Parse the URI - path_part, yaml_path = uri.split(':') - yaml_keys = yaml_path.split('.') + path_part, yaml_path = uri.split(":") + yaml_keys = yaml_path.split(".") - if path_part.startswith('.'): + if path_part.startswith("."): yaml_file_path = caller_dir / f"{path_part[1:].replace('.', '/')}.yaml" else: - yaml_file_path = (PROJ_PATH / path_part.replace('.', '/')).with_suffix('.yaml') + yaml_file_path = (PROJ_PATH / path_part.replace(".", "/")).with_suffix(".yaml") # Load the YAML file - with open(yaml_file_path, 'r') as file: + with open(yaml_file_path, "r") as file: yaml_content = yaml.safe_load(file) # Traverse the YAML content to get the desired template diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 1836566e4..c062aaeab 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -144,16 +144,21 @@ class Config: class DMDockerConf(DockerConf): class Config: - env_prefix = "DM_DOCKER_" + env_prefix = "DM_DOCKER_" build_from_dockerfile: bool = True dockerfile_folder_path: Path = Path(__file__).parent.parent / "scenarios" / "data_mining" / "docker" image: str = "local_dm:latest" mount_path: str = "/workspace/dm_workspace/" default_entry: str = "python train.py" - extra_volumes: dict = {Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/").expanduser().resolve(): "/root/.data/"} + extra_volumes: dict = { + Path("~/.rdagent/.data/physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3/") + .expanduser() + .resolve(): "/root/.data/" + } shm_size: str | None = "16g" + # physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/FIDDLE_mimic3 class DockerEnv(Env[DockerConf]): # TODO: Save the output into a specific file @@ -181,9 +186,9 @@ def _gpu_kwargs(self, client): if not self.conf.enable_gpu: return {} gpu_kwargs = { - "device_requests": [ - docker.types.DeviceRequest(count=-1, capabilities=[['gpu']]) - ] if self.conf.enable_gpu else None, + "device_requests": [docker.types.DeviceRequest(count=-1, capabilities=[["gpu"]])] + if self.conf.enable_gpu + else None, } try: client.containers.run(self.conf.image, "nvidia-smi", **gpu_kwargs) @@ -220,7 +225,7 @@ def run(self, entry: str | None = None, local_path: str | None = None, env: dict # auto_remove=True, # remove too fast might cause the logs not to be get network=self.conf.network, shm_size=self.conf.shm_size, - **self._gpu_kwargs(client) + **self._gpu_kwargs(client), ) logs = container.logs(stream=True) for log in logs: @@ -273,7 +278,9 @@ def prepare(self, username: str, password: str): data_path = next(iter(self.conf.extra_volumes.keys())) if not (Path(data_path)).exists(): logger.info("We are downloading!") - cmd = 'wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/'.format(username, password) + cmd = "wget -r -N -c -np --user={} --password={} -P ~/.rdagent/.data/ https://physionet.org/files/mimic-eicu-fiddle-feature/1.0.0/".format( + username, password + ) os.system(cmd) else: logger.info("Data already exists. Download skipped.") diff --git a/rdagent/utils/workflow.py b/rdagent/utils/workflow.py index 605bfcfa0..63da2dbfb 100644 --- a/rdagent/utils/workflow.py +++ b/rdagent/utils/workflow.py @@ -7,20 +7,19 @@ However, Python generator is not picklable (dill does not support pickle as well) """ -from pathlib import Path +import datetime import pickle -from tqdm.auto import tqdm - - from collections import defaultdict from dataclasses import dataclass, field -import datetime +from pathlib import Path from typing import Callable + +from tqdm.auto import tqdm + from rdagent.log import rdagent_logger as logger class LoopMeta(type): - @staticmethod def _get_steps(bases): """ @@ -28,7 +27,7 @@ def _get_steps(bases): """ steps = [] for base in bases: - steps.extend(LoopMeta._get_steps(base.__bases__) + getattr(base,"steps", [])) + steps.extend(LoopMeta._get_steps(base.__bases__) + getattr(base, "steps", [])) return steps def __new__(cls, clsname, bases, attrs): @@ -52,12 +51,14 @@ class LoopBase: steps: list[Callable] # a list of steps to work on loop_trace: dict[int, list[LoopTrace]] - skip_loop_error: tuple[Exception] = field(default_factory=tuple) # you can define a list of error that will skip current loop + skip_loop_error: tuple[Exception] = field( + default_factory=tuple + ) # you can define a list of error that will skip current loop def __init__(self): - self.loop_idx = 0 # current loop index - self.step_idx = 0 # the index of next step to be run - self.loop_prev_out = {} # the step results of current loop + self.loop_idx = 0 # current loop index + self.step_idx = 0 # the index of next step to be run + self.loop_prev_out = {} # the step results of current loop self.loop_trace = defaultdict(list[LoopTrace]) # the key is the number of loop self.session_folder = logger.log_trace_path / "__session__" @@ -121,7 +122,7 @@ def load(cls, path: str | Path): with path.open("rb") as f: session = pickle.load(f) logger.set_trace_path(session.session_folder.parent) - + max_loop = max(session.loop_trace.keys()) logger.storage.truncate(time=session.loop_trace[max_loop][-1].end) return session diff --git a/test/utils/test_agent_infra.py b/test/utils/test_agent_infra.py index ea1ce25df..4c41293f2 100644 --- a/test/utils/test_agent_infra.py +++ b/test/utils/test_agent_infra.py @@ -5,12 +5,8 @@ from rdagent.utils.agent.tpl import T - - class TestAgentInfra(unittest.TestCase): - def test_agent_infra(self): - # NOTE: It is not serious. It is just for testing sys_prompt = T("components.proposal.prompts:hypothesis_gen.system_prompt").r( targets="targets", diff --git a/test/utils/test_misc.py b/test/utils/test_misc.py index 968b4af3a..5d54770d5 100644 --- a/test/utils/test_misc.py +++ b/test/utils/test_misc.py @@ -9,7 +9,6 @@ def __init__(self, **kwargs): class MiscTest(unittest.TestCase): - def test_singleton(self): a1 = A() a2 = A() From c2681908ec0cad2a4b844d52073b223aa26872e1 Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 00:19:10 +0800 Subject: [PATCH 02/11] fix ci error --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 1bba0c856..a8abb5a65 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ Welcome to RDAgent's documentation! development api_reference policy + test_dropdown/test1 .. test_dropdown/test1 From 7c203854743079ebca83d5b36519c745b47b493f Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 00:20:36 +0800 Subject: [PATCH 03/11] fix ci error --- docs/installation.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index b4852be38..d1adfabb7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -27,13 +27,16 @@ USE_AZURE_TOKEN_PROVIDER ### ☁️ Azure Configuration - Install Azure CLI: - ```sh - curl -L https://aka.ms/InstallAzureCli | bash - ``` + + ```sh + curl -L https://aka.ms/InstallAzureCli | bash + ``` + - Log in to Azure: - ```sh - az login --use-device-code - ``` + + ```sh + az login --use-device-code + ``` - `exit` and re-login to your environment (this step may not be necessary). From c0060ab104a81f0c5673107fb1e54267a9a7fe18 Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 00:27:02 +0800 Subject: [PATCH 04/11] add comments --- .github/workflows/ci.yml | 5 +++-- Makefile | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43b67d49a..e61ffeca6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,11 +20,12 @@ jobs: - run: env | sort - run: make dev - env: + # Two environment variables here can cause sphinx to fail to build in ci. + # CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }} + # CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }} CHAT_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }} CHAT_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} - # CHAT_MAX_TOKENS: ${{ secrets.CHAT_MAX_TOKENS }} CHAT_MODEL: ${{ secrets.CHAT_MODEL }} - # CHAT_TEMPERATURE: ${{ secrets.CHAT_TEMPERATURE }} EMBEDDING_AZURE_API_BASE: ${{ secrets.CHAT_AZURE_API_BASE }} EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }} diff --git a/Makefile b/Makefile index e9a51dd2f..67cf1028c 100644 --- a/Makefile +++ b/Makefile @@ -107,6 +107,7 @@ toml-sort: $(PIPRUN) toml-sort --check pyproject.toml # Check lint with all linters. +# Prioritize fixing isort, then black, otherwise you'll get weird and unfixable black errors. # lint: black isort mypy ruff toml-sort lint: mypy ruff isort black toml-sort sphinx From 1cc4a4395db627d1c30bef4d3c886f666fda3fa7 Mon Sep 17 00:00:00 2001 From: you-n-g Date: Fri, 26 Jul 2024 00:46:37 +0800 Subject: [PATCH 05/11] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 67cf1028c..d4ef5dba4 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ toml-sort: # Check lint with all linters. # Prioritize fixing isort, then black, otherwise you'll get weird and unfixable black errors. -# lint: black isort mypy ruff toml-sort +# lint: mypy ruff lint: mypy ruff isort black toml-sort sphinx # Run pre-commit with autofix against all files. From e99418d3252e9d21983d74003876d9e3a844a353 Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 02:03:09 +0800 Subject: [PATCH 06/11] change sphinx build command --- .github/workflows/ci.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e61ffeca6..75e3c44a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: EMBEDDING_AZURE_API_VERSION: ${{ secrets.CHAT_AZURE_API_VERSION }} EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }} name: lint test docs and build - run: make lint # test docs build sphinx + run: make lint docs-gen # test docs build strategy: matrix: python-version: diff --git a/Makefile b/Makefile index d4ef5dba4..875469dc1 100644 --- a/Makefile +++ b/Makefile @@ -109,7 +109,7 @@ toml-sort: # Check lint with all linters. # Prioritize fixing isort, then black, otherwise you'll get weird and unfixable black errors. # lint: mypy ruff -lint: mypy ruff isort black toml-sort sphinx +lint: mypy ruff isort black toml-sort # Run pre-commit with autofix against all files. pre-commit: From c22f5e5e0f3928b01c1575be7da8f6bbb046bce4 Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 11:05:03 +0800 Subject: [PATCH 07/11] add auto-lint --- Makefile | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 875469dc1..db8d4a3fd 100644 --- a/Makefile +++ b/Makefile @@ -83,9 +83,6 @@ constraints: deepclean black: $(PIPRUN) python -m black --check . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120 -sphinx: - $(PIPRUN) sphinx-build -W --keep-going -b html ./docs _build - # Check lint with isort. isort: $(PIPRUN) python -m isort --check . -s git_ignore_folder -s test/scripts @@ -115,6 +112,25 @@ lint: mypy ruff isort black toml-sort pre-commit: pre-commit run --all-files +######################################################################################## +# Auto Lint +######################################################################################## + +# Auto lint with black. +auto-black: + $(PIPRUN) python -m black . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120 + +# Auto lint with isort. +auto-isort: + $(PIPRUN) python -m isort . -s git_ignore_folder -s test/scripts + +# Auto lint with toml-sort. +auto-toml-sort: + $(PIPRUN) toml-sort pyproject.toml + +# Auto lint with all linters. +auto-lint: auto-isort auto-black auto-toml-sort + ######################################################################################## # Test ######################################################################################## From 085193f8e4ef8283d1c543b9aeeb23252bda988f Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 11:25:46 +0800 Subject: [PATCH 08/11] add black args --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index db8d4a3fd..b3618567f 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,7 @@ constraints: deepclean # Check lint with black. black: - $(PIPRUN) python -m black --check . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120 + $(PIPRUN) python -m black --check --diff . --extend-exclude test/scripts --extend-exclude git_ignore_folder -l 120 # Check lint with isort. isort: From 04ae9c71ca51f9092037e8a58fc6c26e89402c95 Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 11:34:08 +0800 Subject: [PATCH 09/11] format with black --- rdagent/core/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index 1a9d538f5..9a19b6a8f 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -27,9 +27,7 @@ def __new__(cls, *args: Any, **kwargs: Any) -> Any: # TODO: this restriction can be solved. exception_message = "Please only use kwargs in Singleton to avoid misunderstanding." raise RDAgentException(exception_message) - all_args = ( - [(-1, f"{cls.__module__}.{cls.__name__}")] + [(i, args[i]) for i in args] + list(sorted(kwargs.items())) - ) + all_args = [(-1, f"{cls.__module__}.{cls.__name__}")] + [(i, args[i]) for i in args] + list(sorted(kwargs.items())) kwargs_hash = hash(tuple(all_args)) if kwargs_hash not in cls._instance_dict: cls._instance_dict[kwargs_hash] = super().__new__(cls) # Corrected call From 9a5101a33e90d314334999bf8f28fda79aee1ec7 Mon Sep 17 00:00:00 2001 From: Young Date: Fri, 26 Jul 2024 04:00:42 +0000 Subject: [PATCH 10/11] Auto Linting document --- docs/development.rst | 8 +++++++- rdagent/core/utils.py | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 4ab01b064..837f56894 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -11,12 +11,18 @@ For Development make dev ``` -- Run linting and formatting. +- Run linting and checking. ```bash make lint ``` +- Some linting issues can be fixed automatically. We have added a command in the Makefile for easy use. + + ```bash + make auto-lint + ``` + Code Structure ========================= diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index 9a19b6a8f..1a9d538f5 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -27,7 +27,9 @@ def __new__(cls, *args: Any, **kwargs: Any) -> Any: # TODO: this restriction can be solved. exception_message = "Please only use kwargs in Singleton to avoid misunderstanding." raise RDAgentException(exception_message) - all_args = [(-1, f"{cls.__module__}.{cls.__name__}")] + [(i, args[i]) for i in args] + list(sorted(kwargs.items())) + all_args = ( + [(-1, f"{cls.__module__}.{cls.__name__}")] + [(i, args[i]) for i in args] + list(sorted(kwargs.items())) + ) kwargs_hash = hash(tuple(all_args)) if kwargs_hash not in cls._instance_dict: cls._instance_dict[kwargs_hash] = super().__new__(cls) # Corrected call From 95c0e9e306d6888e510435054878ed9baf8274e3 Mon Sep 17 00:00:00 2001 From: SunsetWolf Date: Fri, 26 Jul 2024 12:07:10 +0800 Subject: [PATCH 11/11] fix ci error --- rdagent/core/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index 1a9d538f5..9a19b6a8f 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -27,9 +27,7 @@ def __new__(cls, *args: Any, **kwargs: Any) -> Any: # TODO: this restriction can be solved. exception_message = "Please only use kwargs in Singleton to avoid misunderstanding." raise RDAgentException(exception_message) - all_args = ( - [(-1, f"{cls.__module__}.{cls.__name__}")] + [(i, args[i]) for i in args] + list(sorted(kwargs.items())) - ) + all_args = [(-1, f"{cls.__module__}.{cls.__name__}")] + [(i, args[i]) for i in args] + list(sorted(kwargs.items())) kwargs_hash = hash(tuple(all_args)) if kwargs_hash not in cls._instance_dict: cls._instance_dict[kwargs_hash] = super().__new__(cls) # Corrected call