fuse all code to one commit

feedback.py

@@ -0,0 +1,78 @@
+import json
+from rdagent.oai.llm_utils import APIBackend
+
+def generate_feedback(result, code, hypothesis):
+    # Define the system prompt
+    sys_prompt = (
+        "You are a professional code review assistant. You will receive some code, a result, and a hypothesis. "
+        "Your task is to provide feedback on how well the code and result support or refute the hypothesis. "
+        "Please provide detailed and constructive feedback."
+    )
+
+    # Define the user prompt
+    usr_prompt = (f'''
+        "Given the following hypothesis, result, and code, provide feedback on how well the code and result support or refute the hypothesis. "
+        "Hypothesis: {hypothesis}\n"
+        "Result: {result}\n"
+        "Code:\n```python\n{code}\n```\n"
+        "Please provide detailed and constructive feedback."
+    ''')
+
+    try:
+        # Call the APIBackend to generate the response
+        response = APIBackend().build_messages_and_create_chat_completion(
+            user_prompt=usr_prompt,
+            system_prompt=sys_prompt,
+            json_mode=True,
+        )
+
+        # Log the raw response for debugging
+        print("Raw Response:\n", response)
+
+        # Parse the JSON response to extract the feedback
+        response_json = json.loads(response)
+        feedback = response_json.get("feedback", "No feedback provided")
+
+        print("Generated Feedback:\n", feedback)
+
+        return feedback
+
+    except json.JSONDecodeError as e:
+        print("Error parsing JSON response from LLM:", e)
+    except Exception as e:
+        print("An unexpected error occurred:", e)
+
+def test_generate_feedback():
+    result = "The model achieved an accuracy of 85% on the validation set."
+    code = '''
+import torch
+import torch.nn as nn
+
+class Net(nn.Module):
+    def __init__(self, input_dim, output_dim=1, act="LeakyReLU"):
+        super(Net, self).__init__()
+        self.drop_input = nn.Dropout(0.05)
+        self.fc = nn.Linear(input_dim, output_dim)
+        if act == "LeakyReLU":
+            self.activation = nn.LeakyReLU(negative_slope=0.1, inplace=False)
+        elif act == "SiLU":
+            self.activation = nn.SiLU()
+        else:
+            raise NotImplementedError(f"Activation function {act} is not supported")
+        self.bn = nn.BatchNorm1d(output_dim)
+
+    def forward(self, x):
+        x = self.drop_input(x)
+        x = self.fc(x)
+        x = self.bn(x)
+        x = self.activation(x)
+        return x
+    '''
+    hypothesis = "The data shows time-series quality."
+
+    feedback = generate_feedback(result, code, hypothesis)
+    print("Final Feedback:\n", feedback)
+
+if __name__ == "__main__":
+    test_generate_feedback()
+

rdagent/components/proposal/model_proposal.py

@@ -0,0 +1,97 @@
+from abc import abstractmethod
+from pathlib import Path
+from typing import Tuple
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.model_coder.model import ModelExperiment
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import (
+    Hypothesis,
+    Hypothesis2Experiment,
+    HypothesisGen,
+    HypothesisSet,
+    Scenario,
+    Trace,
+)
+from rdagent.oai.llm_utils import APIBackend
+
+prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+
+ModelHypothesis = Hypothesis
+
+class ModelHypothesisGen(HypothesisGen):
+    def __init__(self, scen: Scenario):
+        super().__init__(scen)
+
+    # The following methods are scenario related so they should be implemented in the subclass
+    @abstractmethod
+    def prepare_context(self, trace: Trace) -> Tuple[dict, bool]:
+        ...
+
+    @abstractmethod
+    def convert_response(self, response: str) -> ModelHypothesis:
+        ...
+
+    def gen(self, trace: Trace) -> ModelHypothesis:
+        context_dict, json_flag = self.prepare_context(trace)
+
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["model_hypothesis_gen"]["system_prompt"])
+            .render(
+                scenario=self.scen.get_scenario_all_desc(),
+                hypothesis_output_format=context_dict["hypothesis_output_format"],
+            )
+        )
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["model_hypothesis_gen"]["user_prompt"])
+            .render(
+                hypothesis_and_feedback=context_dict["hypothesis_and_feedback"],
+                RAG=context_dict["RAG"],
+            )
+        )
+
+        resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
+
+        hypothesis = self.convert_response(resp)
+
+        return hypothesis
+
+
+class ModelHypothesis2Experiment(Hypothesis2Experiment[ModelExperiment]):
+    def __init__(self) -> None:
+        super().__init__()
+
+    @abstractmethod
+    def prepare_context(self, hs: HypothesisSet) -> Tuple[dict, bool]:
+        ...
+
+    @abstractmethod
+    def convert_response(self, response: str) -> ModelExperiment:
+        ...
+
+    def convert(self, hs: HypothesisSet) -> ModelExperiment:
+        context, json_flag = self.prepare_context(hs)
+        system_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["model_hypothesis2experiment"]["system_prompt"])
+            .render(
+                scenario=hs.trace.scen.get_scenario_all_desc(),
+                experiment_output_format=context["experiment_output_format"],
+            )
+        )
+        user_prompt = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["model_hypothesis2experiment"]["user_prompt"])
+            .render(
+                hypothesis_and_feedback=context["hypothesis_and_feedback"],
+                model_list=context["model_list"],
+                RAG=context["RAG"],
+            )
+        )
+
+        resp = APIBackend().build_messages_and_create_chat_completion(user_prompt, system_prompt, json_mode=json_flag)
+
+        return self.convert_response(resp)

rdagent/core/experiment.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Generic, Optional, Sequence, TypeVar
+from typing import Generic, Optional, Sequence, TypeVar, Any
 
 """
 This file contains the all the class about organizing the task in RD-Agent.
@@ -13,9 +13,25 @@ class Task:
     # I think the task version applies to the base class.
     pass
 
-
 ASpecificTask = TypeVar("ASpecificTask", bound=Task)
 
+# class Result():
+
+#     def __init__(self, result_background) -> None:
+#         self.result_background = result_background # scenario actually 
+
+#     @abstractmethod
+#     def build_result(result: str):
+#         """
+#         One pass in the result. It adds up with the background. Then things are processed & stored. 
+#         """
+#         raise NotImplementedError("build_result is not implemented.")
+
+#     # TODO: Find some ways to generalise some qualities of the result. IE. Some backgrounds for it in models? 
+#     # Need to Align 
+
+# ASpecificResult = TypeVar("ASpecificResult", bound=Result)
+
 
 class Implementation(ABC, Generic[ASpecificTask]):
     def __init__(self, target_task: ASpecificTask) -> None:
@@ -119,14 +135,13 @@ class Experiment(ABC, Generic[ASpecificTask, ASpecificImp]):
     The experiment is a sequence of tasks and the implementations of the tasks after generated by the TaskGenerator.
     """
 
-    def __init__(self, sub_tasks: Sequence[ASpecificTask]) -> None:
+    def __init__(self, sub_tasks: Sequence[ASpecificTask], result: Any) -> None:
         self.sub_tasks = sub_tasks
         self.sub_implementations: Sequence[ASpecificImp] = [None for _ in self.sub_tasks]
-
-
+        self.result = result # Note that we are assuming each time we only have ONE Result per EXPERIMENT!! 
+     
 TaskOrExperiment = TypeVar("TaskOrExperiment", Task, Experiment)
 
-
 class Loader(ABC, Generic[TaskOrExperiment]):
     @abstractmethod
     def load(self, *args, **kwargs) -> TaskOrExperiment:

rdagent/core/proposal.py

@@ -3,11 +3,12 @@
 """
 
 from abc import ABC, abstractmethod
-from typing import Dict, Generic, List, Tuple, TypeVar
+from typing import Dict, Generic, List, Tuple, TypeVar, Any
 
 from rdagent.core.evaluation import Feedback
 from rdagent.core.experiment import Experiment
 from rdagent.core.scenario import Scenario
+from rdagent.core.experiment import ASpecificTask
 
 # class data_ana: XXX
 
@@ -29,10 +30,6 @@ def __init__(self, hypothesis: str, reason: str) -> None:
 
 # Origin(path of repo/data/feedback) => view/summarization => generated Hypothesis
 
-
-class HypothesisFeedback(Feedback): ...
-
-
 ASpecificScen = TypeVar("ASpecificScen", bound=Scenario)
 
 
@@ -41,6 +38,12 @@ def __init__(self, scen: ASpecificScen) -> None:
         self.scen: ASpecificScen = scen
         self.hist: list[Tuple[Hypothesis, Experiment, HypothesisFeedback]] = []
 
+    def get_last_experiment_info(self) -> Tuple[Hypothesis, ASpecificTask, Any]:
+        """Access the last experiment result, sub-task, and the corresponding hypothesis."""
+        last_hypothesis, last_experiment, _ = self.hist[-1]
+        last_task = last_experiment.sub_tasks[-1]
+        last_result = last_experiment.result
+        return last_hypothesis, last_task, last_result
 
 class HypothesisGen:
     def __init__(self, scen: Scenario):
@@ -88,6 +91,43 @@ def convert(self, hs: HypothesisSet) -> ASpecificExp:
 
 # Boolean, Reason, Confidence, etc.
 
+
+
+class HypothesisFeedback(Feedback):
+    def __init__(self, observations: str, feedback_for_hypothesis: str, new_hypothesis: str, reasoning: str, attitude: str):
+        self.observations = observations
+        self.feedback_for_hypothesis = feedback_for_hypothesis
+        self.new_hypothesis = new_hypothesis
+        self.reasoning = reasoning
+        self.attitude = attitude
+
+    def __repr__(self):
+        return (f"HypothesisFeedback(observations={self.observations}, "
+                f"feedback_for_hypothesis={self.feedback_for_hypothesis}, "
+                f"new_hypothesis={self.new_hypothesis}, "
+                f"reasoning={self.reasoning}, "
+                f"attitude={self.attitude})")
+
+    def to_dict(self) -> dict:
+        return {
+            "Observations": self.observations,
+            "Feedback for Hypothesis": self.feedback_for_hypothesis,
+            "New Hypothesis": self.new_hypothesis,
+            "Reasoning": self.reasoning,
+            "Attitude": self.attitude
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        return cls(
+            observations=data.get("Observations", ""),
+            feedback_for_hypothesis=data.get("Feedback for Hypothesis", ""),
+            new_hypothesis=data.get("New Hypothesis", ""),
+            reasoning=data.get("Reasoning", ""),
+            attitude=data.get("Attitude", "no")  # Default to "no" if not provided
+        )
+
+
 class HypothesisExperiment2Feedback:
     """ "Generated feedbacks on the hypothesis from **Executed** Implementations of different tasks & their comparisons with previous performances"""
 

rdagent/scenarios/qlib/ model_proposal.py

@@ -0,0 +1,78 @@
+import json
+from pathlib import Path
+from typing import List, Tuple
+
+from jinja2 import Environment, StrictUndefined
+
+from rdagent.components.coder.factor_coder.factor import FactorExperiment, FactorTask
+from rdagent.components.coder.factor_coder.utils import get_data_folder_intro
+from rdagent.components.proposal.factor_proposal import (
+    FactorHypothesis,
+    FactorHypothesis2Experiment,
+    FactorHypothesisGen,
+)
+from rdagent.core.prompts import Prompts
+from rdagent.core.proposal import HypothesisSet, Scenario, Trace
+
+prompt_dict = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+
+QlibFactorHypothesis = FactorHypothesis
+
+
+class QlibFactorHypothesisGen(FactorHypothesisGen):
+    def __init__(self, scen: Scenario) -> Tuple[dict, bool]:
+        super().__init__(scen)
+
+    def prepare_context(self, trace: Trace) -> None:
+        hypothesis_feedback = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["hypothesis_and_feedback"])
+            .render(trace=trace)
+        )
+        context_dict = {
+            "hypothesis_and_feedback": hypothesis_feedback,
+            "RAG": ...,
+            "hypothesis_output_format": prompt_dict["hypothesis_output_format"],
+        }
+        return context_dict, True
+
+    def convert_response(self, response: str) -> FactorHypothesis:
+        response_dict = json.loads(response)
+        hypothesis = QlibFactorHypothesis(hypothesis=response_dict["hypothesis"], reason=response_dict["reason"])
+        return hypothesis
+
+
+class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment):
+    def prepare_context(self, hs: HypothesisSet) -> Tuple[dict | bool]:
+        scenario = hs.trace.scen.get_scenario_all_desc()
+        experiment_output_format = prompt_dict["experiment_output_format"]
+
+        hypothesis_and_feedback = (
+            Environment(undefined=StrictUndefined)
+            .from_string(prompt_dict["hypothesis_and_feedback"])
+            .render(trace=hs.trace)
+        )
+
+        experiment_list: List[FactorExperiment] = [t[1] for t in hs.trace.hist]
+
+        factor_list = []
+        for experiment in experiment_list:
+            factor_list.extend(experiment.sub_tasks)
+
+        return {
+            "scenario": scenario,
+            "hypothesis_and_feedback": hypothesis_and_feedback,
+            "experiment_output_format": experiment_output_format,
+            "factor_list": factor_list,
+            "RAG": ...,
+        }, True
+
+    def convert_response(self, response: str) -> FactorExperiment:
+        response_dict = json.loads(response)
+        tasks = []
+        for factor_name in response_dict:
+            description = response_dict[factor_name]["description"]
+            formulation = response_dict[factor_name]["formulation"]
+            variables = response_dict[factor_name]["variables"]
+            tasks.append(FactorTask(factor_name, description, formulation, variables))
+        return FactorExperiment(tasks)