align benchmark and evolving evaluators

microsoft · Jul 4, 2024 · e626570 · e626570
1 parent d50c9d9
commit e626570
Show file tree

Hide file tree

Showing 7 changed files with 149 additions and 317 deletions.
diff --git a/rdagent/app/qlib_rd_loop/conf.py b/rdagent/app/qlib_rd_loop/conf.py
@@ -7,7 +7,7 @@ class PropSetting(BaseSettings):
     scen: str = "rdagent.scenarios.qlib.experiment.factor_experiment.QlibFactorScenario"
     hypothesis_gen: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesisGen"
     hypothesis2experiment: str = "rdagent.scenarios.qlib.factor_proposal.QlibFactorHypothesis2Experiment"
-    qlib_factor_coder: str = "rdagent.components.coder.factor_coder.CoSTEER.CoSTEERFG"
+    qlib_factor_coder: str = "rdagent.scenarios.qlib.factor_task_implementation.QlibFactorCoSTEER"
     qlib_factor_runner: str = "rdagent.scenarios.qlib.task_generator.data.QlibFactorRunner"
     qlib_factor_summarizer: str = "rdagent.scenarios.qlib.task_generator.feedback.QlibFactorExperiment2Feedback"
 

diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py
@@ -7,13 +7,13 @@
 from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
     FactorCorrelationEvaluator,
+    FactorEqualValueCountEvaluator,
     FactorEvaluator,
     FactorIndexEvaluator,
     FactorIndexFormatEvaluator,
     FactorMissingValuesEvaluator,
     FactorRowCountEvaluator,
     FactorSingleColumnEvaluator,
-    FactorValuesEvaluator,
 )
 from rdagent.components.coder.factor_coder.factor import FileBasedFactorImplementation
 from rdagent.core.exception import ImplementRunException
@@ -94,7 +94,7 @@ def eval_case(
         eval_res = []
         for ev in self.evaluator_l:
             try:
-                eval_res.append((ev, ev.evaluate(case_gt, case_gen)))
+                eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt)))
                 # if the corr ev is successfully evaluated and achieve the best performance, then break
             except ImplementRunException as e:
                 return e
@@ -122,7 +122,7 @@ def __init__(
             FactorRowCountEvaluator(),
             FactorIndexEvaluator(),
             FactorMissingValuesEvaluator(),
-            FactorValuesEvaluator(),
+            FactorEqualValueCountEvaluator(),
             FactorCorrelationEvaluator(hard_check=False),
         ]
         super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs)

diff --git a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py
@@ -3,7 +3,7 @@
 
 from rdagent.components.coder.factor_coder.config import FACTOR_IMPLEMENT_SETTINGS
 from rdagent.components.coder.factor_coder.CoSTEER.evaluators import (
-    FactorEvaluatorV1,
+    FactorEvaluatorForCoder,
     FactorMultiEvaluator,
 )
 from rdagent.components.coder.factor_coder.CoSTEER.evolvable_subjects import (
@@ -45,7 +45,7 @@ def __init__(
         self.knowledge_self_gen = knowledge_self_gen
         self.evolving_strategy = FactorEvolvingStrategyWithGraph()
         # declare the factor evaluator
-        self.factor_evaluator = FactorMultiEvaluator(FactorEvaluatorV1())
+        self.factor_evaluator = FactorMultiEvaluator(FactorEvaluatorForCoder())
         self.evolving_version = 2
 
     def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []):