New Framework for idea proposal and implementation on RD-Agent (#34)

* Commit init framework

* Co-authored-by: Yuante Li (FESCO Adecco Human Resources) <[email protected]>
Co-authored-by: XianBW <[email protected]>

* add an import

* refine the whole framework

* benchmark related framework

* fix black and isort errors

* move requirements to folder

* fix black again

---------

Co-authored-by: Young <[email protected]>
Co-authored-by: xuyang1 <[email protected]>

Makefile

@@ -100,7 +100,7 @@ toml-sort:
 	$(PIPRUN) toml-sort --check pyproject.toml
 
 # Check lint with all linters.
-lint: mypy ruff toml-sort
+lint: black isort mypy ruff toml-sort
 
 # Run pre-commit with autofix against all files.
 pre-commit:

constraints/3.8.txt

@@ -151,4 +151,4 @@ typing_extensions==4.9.0
 tzdata==2023.4
 urllib3==2.1.0
 yarl==1.9.4
-zipp==3.17.0
+zipp==3.17.0

rdagent/app/CI/run.py

@@ -13,6 +13,16 @@
 from typing import Any, Literal
 
 import tree_sitter_python
+from rich import print
+from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn
+from rich.prompt import Prompt
+from rich.rule import Rule
+from rich.syntax import Syntax
+from rich.table import Table
+from rich.text import Text
+from tree_sitter import Language, Node, Parser
+
 from rdagent.core.evolving_framework import (
     Evaluator,
     EvoAgent,
@@ -24,15 +34,6 @@
 )
 from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_utils import APIBackend
-from rich import print
-from rich.panel import Panel
-from rich.progress import Progress, SpinnerColumn, TimeElapsedColumn
-from rich.prompt import Prompt
-from rich.rule import Rule
-from rich.syntax import Syntax
-from rich.table import Table
-from rich.text import Text
-from tree_sitter import Language, Node, Parser
 
 py_parser = Parser(Language(tree_sitter_python.language()))
 CI_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
@@ -355,7 +356,6 @@ def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:  # noqa: ARG002
 
 
 class MypyEvaluator(Evaluator):
-
     def __init__(self, command: str | None = None) -> None:
         if command is None:
             self.command = "mypy . --pretty --no-error-summary --show-column-numbers"
@@ -411,12 +411,10 @@ def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:  # noqa: ARG002
 
 
 class MultiEvaluator(Evaluator):
-
     def __init__(self, *evaluators: Evaluator) -> None:
         self.evaluators = evaluators
 
     def evaluate(self, evo: Repo, **kwargs: Any) -> CIFeedback:
-
         all_errors = defaultdict(list)
         for evaluator in self.evaluators:
             feedback: CIFeedback = evaluator.evaluate(evo, **kwargs)
@@ -438,7 +436,6 @@ def evolve(  # noqa: C901, PLR0912, PLR0915
         knowledge_l: list[Knowledge] | None = None,  # noqa: ARG002
         **kwargs: Any,  # noqa: ARG002
     ) -> Repo:
-
         @dataclass
         class CodeFixGroup:
             start_line: int

rdagent/app/data mining/README.md

@@ -0,0 +1,5 @@
+
+
+
+# TODO
+If we have more efforts, include more scenario.

rdagent/app/factor_extraction_and_implementation/factor_extract_and_implement.py

@@ -1,14 +1,20 @@
 # %%
 from dotenv import load_dotenv
-from rdagent.factor_implementation.CoSTEER import CoSTEERFG
-from rdagent.factor_implementation.task_loader.pdf_loader import FactorImplementationTaskLoaderFromPDFfiles
+
+from rdagent.scenarios.qlib.factor_task_implementation import (
+    COSTEERFG_QUANT_FACTOR_IMPLEMENTATION,
+)
+from rdagent.scenarios.qlib.factor_task_loader.pdf_loader import (
+    FactorImplementationTaskLoaderFromPDFfiles,
+)
 
 assert load_dotenv()
 
 
 def extract_factors_and_implement(report_file_path: str) -> None:
     factor_tasks = FactorImplementationTaskLoaderFromPDFfiles().load(report_file_path)
-    implementation_result = CoSTEERFG().generate(factor_tasks)
+    implementation_result = COSTEERFG_QUANT_FACTOR_IMPLEMENTATION().generate(factor_tasks)
+    # Qlib to run the implementation
     return implementation_result
 
 

rdagent/app/model_implementation/eval.py

@@ -17,7 +17,7 @@
 # TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part.
 # Currently, we just handcraft a workflow for fast evaluation.
 
-mil = ModelImpLoader(DIRNAME.parent.parent / "model_implementation" / "benchmark" /  "gt_code")
+mil = ModelImpLoader(DIRNAME.parent.parent / "model_implementation" / "benchmark" / "gt_code")
 
 mie = ModelImpValEval()
 # Evaluation:

rdagent/app/model_proposal/conf.py

@@ -0,0 +1,12 @@
+from pydantic_settings import BaseSettings
+
+
+class ModelPropSetting(BaseSettings):
+    """"""
+
+    scen: str  # a.b.c:XXXClass
+    # TODO: inital keywards should be included in the settings
+    ...
+
+
+MODEL_PROP_SETTING = ModelPropSetting()

rdagent/app/model_proposal/run.py

@@ -0,0 +1,36 @@
+"""
+TODO: Model Structure RD-Loop
+TODO: move the following code to a new class: Model_RD_Agent
+"""
+
+# import_from
+from rdagent.app.model_proposal.conf import MODEL_PROP_SETTING
+from rdagent.core.implementation import TaskGenerator
+from rdagent.core.proposal import Belief2Task, BeliefSet, Imp2Feedback, Trace
+
+# load_from_cls_uri
+
+
+scen = load_from_cls_uri(MODEL_PROP_SETTING.scen)()
+
+belief_gen = load_from_cls_uri(MODEL_PROP_SETTING.belief_gen)(scen)
+
+belief2task: Belief2Task = load_from_cls_uri(MODEL_PROP_SETTING.belief2task)()
+
+task_gen: TaskGenerator = load_from_cls_uri(MODEL_PROP_SETTING.task_gen)(scen)  # for implementation
+
+imp2feedback: Imp2Feedback = load_from_cls_uri(MODEL_PROP_SETTING.imp2feedback)(scen)  # for implementation
+
+
+iter_n = MODEL_PROP_SETTING.iter_n
+
+trace = Trace()
+
+belief_set = BeliefSet()
+for _ in range(iter_n):
+    belief = belief_gen.gen(trace)
+    task = belief2task.convert(belief)
+    imp = task_gen.gen(task)
+    imp.execute()
+    feedback = imp2feedback.summarize(imp)
+    trace.hist.append((belief, feedback))

rdagent/app/factor_implementation/eval_implement.py → rdagent/app/quant_factor_benchmark/eval.py

@@ -1,7 +1,9 @@
-from rdagent.benchmark.conf import BenchmarkSettings
+from rdagent.components.benchmark.conf import BenchmarkSettings
+from rdagent.components.benchmark.eval_method import FactorImplementEval
 from rdagent.core.utils import import_class
-from rdagent.benchmark.eval_method import FactorImplementEval
-from rdagent.factor_implementation.task_loader.json_loader import FactorTestCaseLoaderFromJsonFile
+from rdagent.scenarios.qlib.factor_task_loader.json_loader import (
+    FactorTestCaseLoaderFromJsonFile,
+)
 
 # 1.read the settings
 bs = BenchmarkSettings()

rdagent/benchmark/conf.py → rdagent/components/benchmark/conf.py

@@ -1,15 +1,16 @@
 from dotenv import load_dotenv
+
 load_dotenv(verbose=True, override=True)
 from dataclasses import field
 from pathlib import Path
-from typing import Optional 
+from typing import Optional
 
 from pydantic_settings import BaseSettings
 
 DIRNAME = Path(__file__).absolute().resolve().parent
 
-class BenchmarkSettings(BaseSettings):
 
+class BenchmarkSettings(BaseSettings):
     ground_truth_dir: Path = DIRNAME / "ground_truth"
 
     bench_data_path: Path = DIRNAME / "example.json"
@@ -22,4 +23,4 @@ class BenchmarkSettings(BaseSettings):
         default_factory=dict,
     )  # extra kwargs for the method to be tested except the task list
 
-    bench_result_path: Path = DIRNAME / "result"
+    bench_result_path: Path = DIRNAME / "result"

rdagent/benchmark/eval_method.py → rdagent/components/benchmark/eval_method.py

@@ -1,27 +1,29 @@
+from collections import defaultdict
 from pathlib import Path
 from typing import List, Tuple, Union
 
 from tqdm import tqdm
-from collections import defaultdict
-from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS
-from rdagent.core.exception import ImplementRunException
-from rdagent.core.task import (
-    TaskImplementation,
-    TestCase,
-)
-from rdagent.factor_implementation.evolving.evaluators import (
+
+from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
     FactorImplementationCorrelationEvaluator,
+    FactorImplementationEvaluator,
     FactorImplementationIndexEvaluator,
     FactorImplementationIndexFormatEvaluator,
     FactorImplementationMissingValuesEvaluator,
     FactorImplementationRowCountEvaluator,
     FactorImplementationSingleColumnEvaluator,
     FactorImplementationValuesEvaluator,
-    FactorImplementationEvaluator,
 )
+from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
+    FileBasedFactorImplementation,
+)
+from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
+from rdagent.core.exception import ImplementRunException
 from rdagent.core.implementation import TaskGenerator
+from rdagent.core.task import TaskImplementation, TestCase
 from rdagent.core.utils import multiprocessing_wrapper
-from rdagent.factor_implementation.evolving.factor import FileBasedFactorImplementation
 
 
 class BaseEval:
@@ -109,14 +111,14 @@ def __init__(
         **kwargs,
     ):
         online_evaluator_l = [
-                FactorImplementationSingleColumnEvaluator(),
-                FactorImplementationIndexFormatEvaluator(),
-                FactorImplementationRowCountEvaluator(),
-                FactorImplementationIndexEvaluator(),
-                FactorImplementationMissingValuesEvaluator(),
-                FactorImplementationValuesEvaluator(),
-                FactorImplementationCorrelationEvaluator(hard_check=False),
-            ]
+            FactorImplementationSingleColumnEvaluator(),
+            FactorImplementationIndexFormatEvaluator(),
+            FactorImplementationRowCountEvaluator(),
+            FactorImplementationIndexEvaluator(),
+            FactorImplementationMissingValuesEvaluator(),
+            FactorImplementationValuesEvaluator(),
+            FactorImplementationCorrelationEvaluator(hard_check=False),
+        ]
         super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)
         self.test_round = test_round
 

rdagent/knowledge_management/graph.py → rdagent/components/knowledge_management/graph.py

@@ -6,8 +6,13 @@
 from pathlib import Path
 from typing import Any, NoReturn
 
+from rdagent.components.knowledge_management.vector_base import (
+    KnowledgeMetaData,
+    PDVectorBase,
+    VectorBase,
+    cosine,
+)
 from rdagent.oai.llm_utils import APIBackend
-from rdagent.knowledge_management.vector_base import KnowledgeMetaData, PDVectorBase, VectorBase, cosine
 
 Node = KnowledgeMetaData
 

rdagent/knowledge_management/vector_base.py → rdagent/components/knowledge_management/vector_base.py

@@ -5,8 +5,8 @@
 import pandas as pd
 from scipy.spatial.distance import cosine
 
-from rdagent.oai.llm_utils import APIBackend
 from rdagent.core.log import RDAgentLog
+from rdagent.oai.llm_utils import APIBackend
 
 
 class KnowledgeMetaData:

rdagent/factor_implementation/CoSTEER.py → rdagent/components/task_implementation/factor_implementation/CoSTEER.py

@@ -1,23 +1,29 @@
 import pickle
 from pathlib import Path
 from typing import List
-from rdagent.core.implementation import TaskGenerator
-from rdagent.core.task import TaskImplementation
-from rdagent.factor_implementation.evolving.knowledge_management import FactorImplementationKnowledgeBaseV1
-from rdagent.factor_implementation.evolving.factor import FactorImplementTask, FactorEvovlingItem
-from rdagent.factor_implementation.evolving.knowledge_management import (
+
+from rdagent.components.task_implementation.factor_implementation.evolving.evaluators import (
+    FactorImplementationEvaluatorV1,
+    FactorImplementationsMultiEvaluator,
+)
+from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
+    FactorEvolvingStrategyWithGraph,
+)
+from rdagent.components.task_implementation.factor_implementation.evolving.factor import (
+    FactorEvovlingItem,
+    FactorImplementTask,
+)
+from rdagent.components.task_implementation.factor_implementation.evolving.knowledge_management import (
     FactorImplementationGraphKnowledgeBase,
     FactorImplementationGraphRAGStrategy,
+    FactorImplementationKnowledgeBaseV1,
 )
-from rdagent.factor_implementation.evolving.evolving_strategy import FactorEvolvingStrategyWithGraph
-from rdagent.factor_implementation.evolving.evaluators import (
-    FactorImplementationsMultiEvaluator,
-    FactorImplementationEvaluatorV1,
-)
-from rdagent.core.evolving_agent import RAGEvoAgent
-from rdagent.factor_implementation.share_modules.factor_implementation_config import (
+from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
     FACTOR_IMPLEMENT_SETTINGS,
 )
+from rdagent.core.evolving_agent import RAGEvoAgent
+from rdagent.core.implementation import TaskGenerator
+from rdagent.core.task import TaskImplementation
 
 
 class CoSTEERFG(TaskGenerator):
@@ -47,7 +53,6 @@ def __init__(
         self.evolving_version = 2
 
     def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):
-
         if former_knowledge_base_path is not None and former_knowledge_base_path.exists():
             factor_knowledge_base = pickle.load(open(former_knowledge_base_path, "rb"))
             if self.evolving_version == 1 and not isinstance(

rdagent/factor_implementation/evolving/evaluators.py → rdagent/components/task_implementation/factor_implementation/evolving/evaluators.py

@@ -1,26 +1,27 @@
 import json
 import re
-
 from abc import abstractmethod
-from typing import Tuple
+from pathlib import Path
+from typing import List, Tuple
 
 import pandas as pd
 from jinja2 import Template
 
-from rdagent.oai.llm_utils import APIBackend
-from rdagent.core.log import RDAgentLog
-from rdagent.factor_implementation.evolving.evolving_strategy import FactorImplementTask, FactorEvovlingItem
-from rdagent.core.task import (
-    TaskImplementation,
+from rdagent.components.task_implementation.factor_implementation.evolving.evolving_strategy import (
+    FactorEvovlingItem,
+    FactorImplementTask,
 )
-from typing import List, Tuple
-from rdagent.core.evolving_framework import QueriedKnowledge, Feedback
+from rdagent.components.task_implementation.factor_implementation.share_modules.factor_implementation_config import (
+    FACTOR_IMPLEMENT_SETTINGS,
+)
+from rdagent.core.conf import RD_AGENT_SETTINGS
 from rdagent.core.evaluation import Evaluator
+from rdagent.core.evolving_framework import Feedback, QueriedKnowledge
+from rdagent.core.log import RDAgentLog
 from rdagent.core.prompts import Prompts
-from rdagent.core.conf import RD_AGENT_SETTINGS
-from rdagent.factor_implementation.share_modules.factor_implementation_config import FACTOR_IMPLEMENT_SETTINGS
+from rdagent.core.task import TaskImplementation
 from rdagent.core.utils import multiprocessing_wrapper
-from pathlib import Path
+from rdagent.oai.llm_utils import APIBackend
 
 evaluate_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")