refine class design and inheritance first version code (#41)

* refine class design and inheritance first version code

* fix all typos

---------

Co-authored-by: xuyang1 <[email protected]>

rdagent/app/CI/run.py

@@ -23,9 +23,9 @@
 from rich.text import Text
 from tree_sitter import Language, Node, Parser
 
+from rdagent.core.evaluation import Evaluator
+from rdagent.core.evolving_agent import EvoAgent
 from rdagent.core.evolving_framework import (
-    Evaluator,
-    EvoAgent,
     EvolvableSubjects,
     EvolvingStrategy,
     EvoStep,

rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py

@@ -1,18 +1,18 @@
 # %%
 from dotenv import load_dotenv
 
+from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
+    FactorImplementationExperimentLoaderFromPDFfiles,
+)
 from rdagent.scenarios.qlib.factor_task_implementation import (
     COSTEERFG_QUANT_FACTOR_IMPLEMENTATION,
 )
-from rdagent.scenarios.qlib.factor_task_loader.pdf_loader import (
-    FactorImplementationTaskLoaderFromPDFfiles,
-)
 
 assert load_dotenv()
 
 
 def extract_factors_and_implement(report_file_path: str) -> None:
-    factor_tasks = FactorImplementationTaskLoaderFromPDFfiles().load(report_file_path)
+    factor_tasks = FactorImplementationExperimentLoaderFromPDFfiles().load(report_file_path)
     implementation_result = COSTEERFG_QUANT_FACTOR_IMPLEMENTATION().generate(factor_tasks)
     # Qlib to run the implementation
     return implementation_result

rdagent/app/model_implementation/eval.py

@@ -2,13 +2,19 @@
 
 DIRNAME = Path(__file__).absolute().resolve().parent
 
-from rdagent.components.task_implementation.model_implementation.benchmark.eval import ModelImpValEval
-from rdagent.components.task_implementation.model_implementation.one_shot import ModelTaskGen
-from rdagent.components.task_implementation.model_implementation.task import  ModelImpLoader, ModelTaskLoderJson
-
+from rdagent.components.task_implementation.model_implementation.benchmark.eval import (
+    ModelImpValEval,
+)
+from rdagent.components.task_implementation.model_implementation.one_shot import (
+    ModelTaskGen,
+)
+from rdagent.components.task_implementation.model_implementation.task import (
+    ModelImpLoader,
+    ModelTaskLoaderJson,
+)
 
 bench_folder = DIRNAME.parent.parent / "components" / "task_implementation" / "model_implementation" / "benchmark"
-mtl = ModelTaskLoderJson(str(bench_folder / "model_dict.json"))
+mtl = ModelTaskLoaderJson(str(bench_folder / "model_dict.json"))
 
 task_l = mtl.load()
 

rdagent/app/model_proposal/run.py

@@ -5,32 +5,37 @@
 
 # import_from
 from rdagent.app.model_proposal.conf import MODEL_PROP_SETTING
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.proposal import Belief2Task, BeliefSet, Imp2Feedback, Trace
+from rdagent.core.proposal import (
+    Experiment2Feedback,
+    Hypothesis2Experiment,
+    HypothesisSet,
+    Trace,
+)
+from rdagent.core.task_generator import TaskGenerator
 
 # load_from_cls_uri
 
 
 scen = load_from_cls_uri(MODEL_PROP_SETTING.scen)()
 
-belief_gen = load_from_cls_uri(MODEL_PROP_SETTING.belief_gen)(scen)
+hypothesis_gen = load_from_cls_uri(MODEL_PROP_SETTING.hypothesis_gen)(scen)
 
-belief2task: Belief2Task = load_from_cls_uri(MODEL_PROP_SETTING.belief2task)()
+hypothesis2task: Hypothesis2Experiment = load_from_cls_uri(MODEL_PROP_SETTING.hypothesis2task)()
 
 task_gen: TaskGenerator = load_from_cls_uri(MODEL_PROP_SETTING.task_gen)(scen)  # for implementation
 
-imp2feedback: Imp2Feedback = load_from_cls_uri(MODEL_PROP_SETTING.imp2feedback)(scen)  # for implementation
+imp2feedback: Experiment2Feedback = load_from_cls_uri(MODEL_PROP_SETTING.imp2feedback)(scen)  # for implementation
 
 
 iter_n = MODEL_PROP_SETTING.iter_n
 
 trace = Trace()
 
-belief_set = BeliefSet()
+hypothesis_set = HypothesisSet()
 for _ in range(iter_n):
-    belief = belief_gen.gen(trace)
-    task = belief2task.convert(belief)
+    hypothesis = hypothesis_gen.gen(trace)
+    task = hypothesis2task.convert(hypothesis)
     imp = task_gen.gen(task)
     imp.execute()
     feedback = imp2feedback.summarize(imp)
-    trace.hist.append((belief, feedback))
+    trace.hist.append((hypothesis, feedback))

rdagent/components/benchmark/eval_method.py

@@ -4,6 +4,9 @@
 
 from tqdm import tqdm
 
+from rdagent.components.task_implementation.factor_implementation.config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
 from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
     FactorImplementationCorrelationEvaluator,
     FactorImplementationEvaluator,
@@ -14,18 +17,25 @@
     FactorImplementationSingleColumnEvaluator,
     FactorImplementationValuesEvaluator,
 )
-from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
+from rdagent.components.task_implementation.factor_implementation.factor import (
     FileBasedFactorImplementation,
 )
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
-)
 from rdagent.core.exception import ImplementRunException
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.task import TaskImplementation, TestCase
+from rdagent.core.experiment import Implementation, Task
+from rdagent.core.task_generator import TaskGenerator
 from rdagent.core.utils import multiprocessing_wrapper
 
 
+class TestCase:
+    def __init__(
+        self,
+        target_task: list[Task] = [],
+        ground_truth: list[Implementation] = [],
+    ):
+        self.ground_truth = ground_truth
+        self.target_task = target_task
+
+
 class BaseEval:
     """
     The benchmark benchmark evaluation.
@@ -56,7 +66,7 @@ def load_cases_to_eval(
         self,
         path: Union[Path, str],
         **kwargs,
-    ) -> List[TaskImplementation]:
+    ) -> List[Implementation]:
         path = Path(path)
         fi_l = []
         for tc in self.test_cases:
@@ -69,8 +79,8 @@ def load_cases_to_eval(
 
     def eval_case(
         self,
-        case_gt: TaskImplementation,
-        case_gen: TaskImplementation,
+        case_gt: Implementation,
+        case_gen: Implementation,
     ) -> List[Union[Tuple[FactorImplementationEvaluator, object], Exception]]:
         """Parameters
         ----------
@@ -137,11 +147,11 @@ def eval(self):
                 print("Manually interrupted the evaluation. Saving existing results")
                 break
 
-            if len(gen_factor_l.corresponding_implementations) != len(self.test_cases.ground_truth):
+            if len(gen_factor_l.sub_implementations) != len(self.test_cases.ground_truth):
                 raise ValueError(
                     "The number of cases to eval should be equal to the number of test cases.",
                 )
-            gen_factor_l_all_rounds.extend(gen_factor_l.corresponding_implementations)
+            gen_factor_l_all_rounds.extend(gen_factor_l.sub_implementations)
             test_cases_all_rounds.extend(self.test_cases.ground_truth)
 
         eval_res_list = multiprocessing_wrapper(

rdagent/components/idea_proposal/factor_proposal.py

@@ -0,0 +1,61 @@
+from abc import abstractmethod
+from pathlib import Path
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import (
+    Hypothesis,
+    Hypothesis2Task,
+    HypothesisGen,
+    Scenario,
+    Trace,
+)
+from rdagent.oai.llm_utils import APIBackend
+
+prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+
+
+FactorHypothesis = Hypothesis
+
+
+class FactorHypothesisGen(HypothesisGen):
+    def __init__(self, scen: Scenario):
+        super().__init__(scen)
+        self.gen_context_flag = False
+        self.gen_context_dict = None
+        self.gen_json_flag = False
+
+    # The following methods are scenario related so they should be implemented in the subclass
+    @abstractmethod
+    def prepare_gen_context(self, trace: Trace) -> None: ...
+
+    @abstractmethod
+    def gen_response_to_hypothesis_list(self, response: str) -> FactorHypothesis: ...
+
+    def gen(self, trace: Trace) -> FactorHypothesis:
+        assert self.gen_context_flag, "Please call prepare_gen_context before calling gen."
+        self.gen_context_flag = False  # reset the flag
+
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["factor_hypothesis_gen"]["system_prompt"])
+            .render(scenario=self.scen.get_scenario_all_desc())
+        )
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["factor_hypothesis_gen"]["user_prompt"])
+            .render(self.gen_context_dict)
+        )
+
+        resp = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt, system_prompt, json_mode=self.gen_json_flag
+        )
+
+        hypothesis = self.gen_response_to_hypothesis_list(resp)
+
+        return hypothesis
+
+
+class FactorHypothesis2Task(Hypothesis2Task):
+    def convert(self, bs: FactorHypothesis) -> None: ...

rdagent/components/idea_proposal/prompts.yaml

@@ -0,0 +1,22 @@
+factor_hypothesis_gen:
+  system_prompt: |-
+    The user is trying to generate new hypothesis on the factors in data-driven research and development.
+    The factors are used in a certain scenario, the scenario is as follows:
+    {{ scenario }}
+    The user has made several hypothesis on this sencario and did several evaluation on them. The user will provide this information to you.
+    To help you generate new hypothesis, the user has prepared some additional information for you. You should use this information to help generate new factors.
+  user_prompt: |-
+    The user has made several hypothesis on this sencario and did several evaluation on them.
+    The former hypothesis and the corresponding feedbacks are as follows:
+    {{ hypothesis_and_feedback }}
+    To help you generate new factors, we have prepared the following information for you:
+    {{ RAG }}
+    Please generate the new hypothesis based on the information above and generate the output following the format below:
+    {{ factor_output_format }}
+
+factor_hypothesis_to_tasks:
+  system_prompt: |-
+    The user is trying to generate new factors based on the hypothesis generated in the previous step. 
+    The factors are used in certain scenario, the scenario is as follows:
+    {{ scenario }}
+

rdagent/components/loader/experiment_loader.py

@@ -0,0 +1,12 @@
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorExperiment,
+)
+from rdagent.core.experiment import Loader
+
+
+class FactorExperimentLoader(Loader[FactorExperiment]):
+    pass
+
+
+class ModelExperimentLoader(Loader[FactorExperiment]):
+    pass

rdagent/components/loader/task_loader.py

@@ -0,0 +1,12 @@
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorTask,
+)
+from rdagent.core.experiment import Loader
+
+
+class FactorTaskLoader(Loader[FactorTask]):
+    pass
+
+
+class ModelTaskLoader(Loader[FactorTask]):
+    pass

rdagent/components/task_implementation/factor_implementation/CoSTEER.py

@@ -2,31 +2,33 @@
 from pathlib import Path
 from typing import List
 
+from rdagent.components.task_implementation.factor_implementation.config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
 from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
     FactorImplementationEvaluatorV1,
     FactorImplementationsMultiEvaluator,
 )
+from rdagent.components.task_implementation.factor_implementation.evolving.evolvable_subjects import (
+    FactorEvolvingItem,
+)
 from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
     FactorEvolvingStrategyWithGraph,
 )
-from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
-    FactorEvovlingItem,
-    FactorImplementTask,
-)
 from rdagent.components.task_implementation.factor_implementation.evolving.knowledge_management import (
     FactorImplementationGraphKnowledgeBase,
     FactorImplementationGraphRAGStrategy,
     FactorImplementationKnowledgeBaseV1,
 )
-from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
-    FACTOR_IMPLEMENT_SETTINGS,
+from rdagent.components.task_implementation.factor_implementation.factor import (
+    FactorExperiment,
 )
 from rdagent.core.evolving_agent import RAGEvoAgent
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.task import TaskImplementation
+from rdagent.core.experiment import Experiment
+from rdagent.core.task_generator import TaskGenerator
 
 
-class CoSTEERFG(TaskGenerator):
+class CoSTEERFG(TaskGenerator[FactorExperiment]):
     def __init__(
         self,
         with_knowledge: bool = True,
@@ -74,7 +76,7 @@ def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, c
             )
         return factor_knowledge_base
 
-    def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]:
+    def generate(self, exp: FactorExperiment) -> FactorExperiment:
         # init knowledge base
         factor_knowledge_base = self.load_or_init_knowledge_base(
             former_knowledge_base_path=self.knowledge_base_path,
@@ -83,8 +85,8 @@ def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]
         # init rag method
         self.rag = FactorImplementationGraphRAGStrategy(factor_knowledge_base)
 
-        # init indermediate items
-        factor_implementations = FactorEvovlingItem(target_factor_tasks=tasks)
+        # init intermediate items
+        factor_implementations = FactorEvolvingItem(sub_tasks=exp.sub_tasks)
 
         self.evolve_agent = RAGEvoAgent(max_loop=self.max_loop, evolving_strategy=self.evolving_strategy, rag=self.rag)
 
@@ -100,5 +102,5 @@ def generate(self, tasks: List[FactorImplementTask]) -> List[TaskImplementation]
         if self.new_knowledge_base_path is not None:
             pickle.dump(factor_knowledge_base, open(self.new_knowledge_base_path, "wb"))
         self.knowledge_base = factor_knowledge_base
-        self.latest_factor_implementations = tasks
+        self.latest_factor_implementations = exp.sub_tasks
         return factor_implementations