From 711c5e68094d09092abc642455336c5ced246eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Sun, 18 Feb 2024 19:16:42 +0800 Subject: [PATCH 01/24] set `block on white` style for markdown block. --- metagpt/actions/mi/execute_nb_code.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index a8c9c30859..cb3c80352e 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -226,22 +226,24 @@ def display_markdown(content: str): matches = re.finditer(r"```(.+?)```", content, re.DOTALL) start_index = 0 content_panels = [] + # 文本背景色和文字颜色设置 + style = "black on white" # 逐个打印匹配到的文本和代码 for match in matches: text_content = content[start_index : match.start()].strip() code_content = match.group(0).strip()[3:-3] # Remove triple backticks if text_content: - content_panels.append(Panel(Markdown(text_content), box=MINIMAL)) + content_panels.append(Panel(Markdown(text_content), style=style, box=MINIMAL)) if code_content: - content_panels.append(Panel(Markdown(f"```{code_content}"), box=MINIMAL)) + content_panels.append(Panel(Markdown(f"```{code_content}"), style=style, box=MINIMAL)) start_index = match.end() # 打印剩余文本（如果有） remaining_text = content[start_index:].strip() if remaining_text: - content_panels.append(Panel(Markdown(remaining_text), box=MINIMAL)) + content_panels.append(Panel(Markdown(remaining_text), style=style, box=MINIMAL)) # 在Live模式中显示所有Panel with Live(auto_refresh=False, console=Console(), vertical_overflow="visible") as live: From e1eb69b07d4cec1b3714279d3fe866f397f7109a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Mon, 19 Feb 2024 10:21:59 +0800 Subject: [PATCH 02/24] chore: annotated in English. --- metagpt/actions/mi/execute_nb_code.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index cb3c80352e..8e8e997b83 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -215,20 +215,21 @@ def truncate(result: str, keep_len: int = 2000, is_success: bool = True): def remove_escape_and_color_codes(input_str: str): - # 使用正则表达式去除转义字符和颜色代码 + # 使用正则表达式去除jupyter notebook输出结果中的转义字符和颜色代码 + # Use regular expressions to get rid of escape characters and color codes in jupyter notebook output. pattern = re.compile(r"\x1b\[[0-9;]*[mK]") result = pattern.sub("", input_str) return result def display_markdown(content: str): - # 使用正则表达式逐个匹配代码块 + # Use regular expressions to match blocks of code one by one. matches = re.finditer(r"```(.+?)```", content, re.DOTALL) start_index = 0 content_panels = [] - # 文本背景色和文字颜色设置 + # Set the text background color and text color. style = "black on white" - # 逐个打印匹配到的文本和代码 + # Print the matching text and code one by one. for match in matches: text_content = content[start_index : match.start()].strip() code_content = match.group(0).strip()[3:-3] # Remove triple backticks @@ -240,12 +241,12 @@ def display_markdown(content: str): content_panels.append(Panel(Markdown(f"```{code_content}"), style=style, box=MINIMAL)) start_index = match.end() - # 打印剩余文本（如果有） + # Print remaining text (if any). remaining_text = content[start_index:].strip() if remaining_text: content_panels.append(Panel(Markdown(remaining_text), style=style, box=MINIMAL)) - # 在Live模式中显示所有Panel + # Display all panels in Live mode. with Live(auto_refresh=False, console=Console(), vertical_overflow="visible") as live: live.update(Group(*content_panels)) live.refresh() From 3dde4664f40f9ae9f92441369a7cab0c88f7e68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 21 Feb 2024 11:03:32 +0800 Subject: [PATCH 03/24] add sales_forecast in machine_learning. --- examples/mi/machine_learning.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index a8ab5051e0..a76561a379 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -3,10 +3,36 @@ from metagpt.roles.mi.interpreter import Interpreter -async def main(auto_run: bool = True): - requirement = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." +DATA_DIR = "examples/mi/data" +requirements = { + "wine": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", + + # sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data + "sales_forecast": f""" + # Goal + Use time series regression machine learning to make predictions for Dept sales of the stores as accurate as possible. + + # Datasets Available + - train_data: {DATA_DIR}/WalmartSalesForecast/new_train.csv + - test_data: {DATA_DIR}/WalmartSalesForecast/new_test.csv + - additional data: {DATA_DIR}/WalmartSalesForecast/features.csv; To merge on train, test data. + - stores data: {DATA_DIR}/WalmartSalesForecast/stores.csv; To merge on train, test data. + + # Metric + The metric of the competition is weighted mean absolute error (WMAE) for test data. + + # Notice + - *print* key variables to get more information for next task step. + - Perform data analysis by plotting sales trends, holiday effects, distribution of sales across stores/departments using box/violin on the train data. + - Make sure the DataFrame.dtypes must be int, float or bool, and drop date column. + - Plot scatter plots of groud truth and predictions on test data. + """ +} + + +async def main(auto_run: bool = True, use_case: str = 'wine'): mi = Interpreter(auto_run=auto_run) - await mi.run(requirement) + await mi.run(requirements[use_case]) if __name__ == "__main__": From 247fa13e864b22848e33656b21752e64db30cc11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 21 Feb 2024 11:03:48 +0800 Subject: [PATCH 04/24] fix: pip error. --- metagpt/actions/mi/execute_nb_code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index 8e8e997b83..0e4563a375 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -182,7 +182,7 @@ async def run(self, code: str, language: Literal["python", "markdown"] = "python outputs = self.parse_outputs(self.nb.cells[-1].outputs) outputs, success = truncate(remove_escape_and_color_codes(outputs), is_success=success) - if "!pip" in outputs: + if "!pip" in code: success = False return outputs, success From 662fbd7e5554f426cc34e42402e48fc5ab407621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 21 Feb 2024 11:43:02 +0800 Subject: [PATCH 05/24] chore. --- examples/mi/machine_learning.py | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index a76561a379..689335db39 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -2,35 +2,35 @@ from metagpt.roles.mi.interpreter import Interpreter - DATA_DIR = "examples/mi/data" -requirements = { - "wine": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", +WINE_REQ = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." + +# sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data, +# new_train, new_test from train.csv. +SALES_FORECAST_REQ = f""" +# Goal +Use time series regression machine learning to make predictions for Dept sales of the stores as accurate as possible. - # sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data - "sales_forecast": f""" - # Goal - Use time series regression machine learning to make predictions for Dept sales of the stores as accurate as possible. +# Datasets Available +- train_data: {DATA_DIR}/WalmartSalesForecast/new_train.csv +- test_data: {DATA_DIR}/WalmartSalesForecast/new_test.csv +- additional data: {DATA_DIR}/WalmartSalesForecast/features.csv; To merge on train, test data. +- stores data: {DATA_DIR}/WalmartSalesForecast/stores.csv; To merge on train, test data. - # Datasets Available - - train_data: {DATA_DIR}/WalmartSalesForecast/new_train.csv - - test_data: {DATA_DIR}/WalmartSalesForecast/new_test.csv - - additional data: {DATA_DIR}/WalmartSalesForecast/features.csv; To merge on train, test data. - - stores data: {DATA_DIR}/WalmartSalesForecast/stores.csv; To merge on train, test data. +# Metric +The metric of the competition is weighted mean absolute error (WMAE) for test data. - # Metric - The metric of the competition is weighted mean absolute error (WMAE) for test data. +# Notice +- *print* key variables to get more information for next task step. +- Perform data analysis by plotting sales trends, holiday effects, distribution of sales across stores/departments using box/violin on the train data. +- Make sure the DataFrame.dtypes must be int, float or bool, and drop date column. +- Plot scatter plots of groud truth and predictions on test data. +""" - # Notice - - *print* key variables to get more information for next task step. - - Perform data analysis by plotting sales trends, holiday effects, distribution of sales across stores/departments using box/violin on the train data. - - Make sure the DataFrame.dtypes must be int, float or bool, and drop date column. - - Plot scatter plots of groud truth and predictions on test data. - """ -} +requirements = {"wine": WINE_REQ, "sales_forecast": SALES_FORECAST_REQ} -async def main(auto_run: bool = True, use_case: str = 'wine'): +async def main(auto_run: bool = True, use_case: str = "wine"): mi = Interpreter(auto_run=auto_run) await mi.run(requirements[use_case]) From fc4017480205104f281f0367ef83acc433375a59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 21 Feb 2024 21:33:11 +0800 Subject: [PATCH 06/24] chore. --- examples/mi/machine_learning.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index 689335db39..5f9d5b0cde 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -2,29 +2,30 @@ from metagpt.roles.mi.interpreter import Interpreter -DATA_DIR = "examples/mi/data" WINE_REQ = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." -# sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data, -# new_train, new_test from train.csv. +# DATA_DIR = "your/path/to/data" +DATA_DIR = "examples/mi/data/WalmartSalesForecast2" +# sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data SALES_FORECAST_REQ = f""" # Goal -Use time series regression machine learning to make predictions for Dept sales of the stores as accurate as possible. +Train a model to predict sales for each department in every store (split the last 40 weeks records as validation dataset, +the others is train dataset), include plot sales trends, holiday effects, distribution of sales across stores/departments, +using box on the train dataset, print metric and plot scatter plots of groud truth and predictions on validation data. +save predictions on test data. # Datasets Available -- train_data: {DATA_DIR}/WalmartSalesForecast/new_train.csv -- test_data: {DATA_DIR}/WalmartSalesForecast/new_test.csv -- additional data: {DATA_DIR}/WalmartSalesForecast/features.csv; To merge on train, test data. -- stores data: {DATA_DIR}/WalmartSalesForecast/stores.csv; To merge on train, test data. +- train_data: {DATA_DIR}/train.csv +- test_data: {DATA_DIR}/test.csv, no label data. +- additional data: {DATA_DIR}/features.csv +- stores data: {DATA_DIR}/stores.csv # Metric The metric of the competition is weighted mean absolute error (WMAE) for test data. # Notice - *print* key variables to get more information for next task step. -- Perform data analysis by plotting sales trends, holiday effects, distribution of sales across stores/departments using box/violin on the train data. -- Make sure the DataFrame.dtypes must be int, float or bool, and drop date column. -- Plot scatter plots of groud truth and predictions on test data. +- Only When you fit the model, make the DataFrame.dtypes to be int, float or bool, and drop date column. """ requirements = {"wine": WINE_REQ, "sales_forecast": SALES_FORECAST_REQ} @@ -32,7 +33,12 @@ async def main(auto_run: bool = True, use_case: str = "wine"): mi = Interpreter(auto_run=auto_run) - await mi.run(requirements[use_case]) + if use_case == "wine": + requirement = requirements[use_case] + else: + assert DATA_DIR != "your/path/to/data", f"Please set DATA_DIR for the use_case: {use_case}!" + requirement = requirements[use_case] + await mi.run(requirement) if __name__ == "__main__": From 6652aa09ce8e2e19ba4a8ffd89013fae2fccb23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 21 Feb 2024 22:52:18 +0800 Subject: [PATCH 07/24] delete feature.csv, store.csv, test.csv. --- examples/mi/machine_learning.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index 5f9d5b0cde..652e7c9089 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -10,22 +10,17 @@ SALES_FORECAST_REQ = f""" # Goal Train a model to predict sales for each department in every store (split the last 40 weeks records as validation dataset, -the others is train dataset), include plot sales trends, holiday effects, distribution of sales across stores/departments, -using box on the train dataset, print metric and plot scatter plots of groud truth and predictions on validation data. -save predictions on test data. +the others is train dataset), include plot sales trends, print metric and plot scatter plots of +groud truth and predictions on validation data. # Datasets Available - train_data: {DATA_DIR}/train.csv -- test_data: {DATA_DIR}/test.csv, no label data. -- additional data: {DATA_DIR}/features.csv -- stores data: {DATA_DIR}/stores.csv # Metric The metric of the competition is weighted mean absolute error (WMAE) for test data. # Notice - *print* key variables to get more information for next task step. -- Only When you fit the model, make the DataFrame.dtypes to be int, float or bool, and drop date column. """ requirements = {"wine": WINE_REQ, "sales_forecast": SALES_FORECAST_REQ} @@ -36,6 +31,7 @@ async def main(auto_run: bool = True, use_case: str = "wine"): if use_case == "wine": requirement = requirements[use_case] else: + mi.use_tools = True assert DATA_DIR != "your/path/to/data", f"Please set DATA_DIR for the use_case: {use_case}!" requirement = requirements[use_case] await mi.run(requirement) From 2d8906091824263236db19ade7c1a9edfb5146af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 21 Feb 2024 23:03:36 +0800 Subject: [PATCH 08/24] chore. --- examples/mi/machine_learning.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index 652e7c9089..9e4045bbb4 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -23,17 +23,17 @@ - *print* key variables to get more information for next task step. """ -requirements = {"wine": WINE_REQ, "sales_forecast": SALES_FORECAST_REQ} +REQUIREMENTS = {"wine": WINE_REQ, "sales_forecast": SALES_FORECAST_REQ} async def main(auto_run: bool = True, use_case: str = "wine"): mi = Interpreter(auto_run=auto_run) if use_case == "wine": - requirement = requirements[use_case] + requirement = REQUIREMENTS[use_case] else: mi.use_tools = True assert DATA_DIR != "your/path/to/data", f"Please set DATA_DIR for the use_case: {use_case}!" - requirement = requirements[use_case] + requirement = REQUIREMENTS[use_case] await mi.run(requirement) From bd994adca2ec4b8199d75ea99365881f69ebb557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Wed, 21 Feb 2024 23:07:42 +0800 Subject: [PATCH 09/24] chore. --- examples/mi/machine_learning.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index 9e4045bbb4..43f1a1d3a2 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -4,8 +4,7 @@ WINE_REQ = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." -# DATA_DIR = "your/path/to/data" -DATA_DIR = "examples/mi/data/WalmartSalesForecast2" +DATA_DIR = "your/path/to/data" # sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data SALES_FORECAST_REQ = f""" # Goal From ec95cedb368c5014c2978d65966e1aaee7639669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Thu, 22 Feb 2024 21:07:32 +0800 Subject: [PATCH 10/24] update SALES_FORECAST_REQ. --- examples/mi/machine_learning.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index 43f1a1d3a2..d67a427129 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -6,20 +6,8 @@ DATA_DIR = "your/path/to/data" # sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data -SALES_FORECAST_REQ = f""" -# Goal -Train a model to predict sales for each department in every store (split the last 40 weeks records as validation dataset, -the others is train dataset), include plot sales trends, print metric and plot scatter plots of -groud truth and predictions on validation data. - -# Datasets Available -- train_data: {DATA_DIR}/train.csv - -# Metric -The metric of the competition is weighted mean absolute error (WMAE) for test data. - -# Notice -- *print* key variables to get more information for next task step. +SALES_FORECAST_REQ = f"""Train a model to predict sales for each department in every store (split the last 40 weeks records as validation dataset, the others is train dataset), include plot sales trends, print metric and plot scatter plots of +groud truth and predictions on validation data. Datasets Available is train_data: {DATA_DIR}/train.csv, The metric of the competition is weighted mean absolute error (WMAE) for test data. Notice: *print* key variables to get more information for next task step. """ REQUIREMENTS = {"wine": WINE_REQ, "sales_forecast": SALES_FORECAST_REQ} From 08e00b4dc2894dc233bbb2802009f9f070092b3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Thu, 22 Feb 2024 21:48:05 +0800 Subject: [PATCH 11/24] chore: simplify code. --- examples/mi/machine_learning.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index d67a427129..53168e0e9e 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -4,10 +4,10 @@ WINE_REQ = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." -DATA_DIR = "your/path/to/data" +DATA_DIR = "examples/mi/data/WalmartSalesForecast2" # sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data -SALES_FORECAST_REQ = f"""Train a model to predict sales for each department in every store (split the last 40 weeks records as validation dataset, the others is train dataset), include plot sales trends, print metric and plot scatter plots of -groud truth and predictions on validation data. Datasets Available is train_data: {DATA_DIR}/train.csv, The metric of the competition is weighted mean absolute error (WMAE) for test data. Notice: *print* key variables to get more information for next task step. +SALES_FORECAST_REQ = f"""Train a model to predict sales for each department in every store (split the last 40 weeks records as validation dataset, the others is train dataset), include plot total sales trends, print metric and plot scatter plots of +groud truth and predictions on validation data. Dataset is {DATA_DIR}/train.csv, the metric is weighted mean absolute error (WMAE) for test data. Notice: *print* key variables to get more information for next task step. """ REQUIREMENTS = {"wine": WINE_REQ, "sales_forecast": SALES_FORECAST_REQ} @@ -15,12 +15,7 @@ async def main(auto_run: bool = True, use_case: str = "wine"): mi = Interpreter(auto_run=auto_run) - if use_case == "wine": - requirement = REQUIREMENTS[use_case] - else: - mi.use_tools = True - assert DATA_DIR != "your/path/to/data", f"Please set DATA_DIR for the use_case: {use_case}!" - requirement = REQUIREMENTS[use_case] + requirement = REQUIREMENTS[use_case] await mi.run(requirement) From 041a2d61097bad45b5b7e28a880b37268a285c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Thu, 22 Feb 2024 21:56:02 +0800 Subject: [PATCH 12/24] chore. --- examples/mi/machine_learning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mi/machine_learning.py b/examples/mi/machine_learning.py index 53168e0e9e..56c68f69ee 100644 --- a/examples/mi/machine_learning.py +++ b/examples/mi/machine_learning.py @@ -4,7 +4,7 @@ WINE_REQ = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." -DATA_DIR = "examples/mi/data/WalmartSalesForecast2" +DATA_DIR = "path/to/your/data" # sales_forecast data from https://www.kaggle.com/datasets/aslanahmedov/walmart-sales-forecast/data SALES_FORECAST_REQ = f"""Train a model to predict sales for each department in every store (split the last 40 weeks records as validation dataset, the others is train dataset), include plot total sales trends, print metric and plot scatter plots of groud truth and predictions on validation data. Dataset is {DATA_DIR}/train.csv, the metric is weighted mean absolute error (WMAE) for test data. Notice: *print* key variables to get more information for next task step. From b5a534d617f9eed45ccd8c36c2ddabe22dfce286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Sun, 10 Mar 2024 22:18:13 +0800 Subject: [PATCH 13/24] fix: ipykernel_error that is `Too many open files in system` --- metagpt/roles/role.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 3938664ba3..73edfc52b8 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -503,6 +503,9 @@ async def _plan_and_act(self) -> Message: self.rc.memory.add(rsp) # add to persistent memory + if hasattr(self, "execute_code") and hasattr(self.execute_code, "terminate"): + await self.execute_code.terminate() + return rsp async def _act_on_task(self, current_task: Task) -> TaskResult: From 1e04e3410346efafc4c8efcf35700650a2ad5608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Sun, 10 Mar 2024 23:52:19 +0800 Subject: [PATCH 14/24] add test. --- tests/metagpt/actions/mi/test_execute_nb_code.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/metagpt/actions/mi/test_execute_nb_code.py b/tests/metagpt/actions/mi/test_execute_nb_code.py index 59a8140540..3059ad3ae1 100644 --- a/tests/metagpt/actions/mi/test_execute_nb_code.py +++ b/tests/metagpt/actions/mi/test_execute_nb_code.py @@ -104,6 +104,15 @@ async def test_terminate(): time.sleep(2) assert executor.nb_client.km is None + for _ in range(200): + executor = ExecuteNbCode() + await executor.run(code='print("This is a code!")', language="python") + is_kernel_alive = await executor.nb_client.km.is_alive() + assert is_kernel_alive + await executor.terminate() + assert executor.nb_client.km is None + assert executor.nb_client.kc is None + await executor.terminate() @pytest.mark.asyncio From 803359a7588f5d955395222368320e450cc27bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Mon, 11 Mar 2024 11:25:23 +0800 Subject: [PATCH 15/24] refine: add terminate in Interpreter instead of Role. --- metagpt/actions/mi/execute_nb_code.py | 3 ++- metagpt/roles/mi/interpreter.py | 4 ++++ metagpt/roles/role.py | 3 --- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index 0e4563a375..3a64a8bec1 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -58,7 +58,8 @@ async def build(self): async def terminate(self): """kill NotebookClient""" - await self.nb_client._async_cleanup_kernel() + if self.nb_client.km is not None: + await self.nb_client._async_cleanup_kernel() async def reset(self): """reset NotebookClient""" diff --git a/metagpt/roles/mi/interpreter.py b/metagpt/roles/mi/interpreter.py index fa50098e9d..2392863a00 100644 --- a/metagpt/roles/mi/interpreter.py +++ b/metagpt/roles/mi/interpreter.py @@ -42,6 +42,10 @@ def __init__( def working_memory(self): return self.rc.working_memory + async def _plan_and_act(self) -> Message: + await super()._plan_and_act() + await self.execute_code.terminate() + async def _act_on_task(self, current_task: Task) -> TaskResult: code, result, is_success = await self._write_and_exec_code() task_result = TaskResult(code=code, result=result, is_success=is_success) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 73edfc52b8..3938664ba3 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -503,9 +503,6 @@ async def _plan_and_act(self) -> Message: self.rc.memory.add(rsp) # add to persistent memory - if hasattr(self, "execute_code") and hasattr(self.execute_code, "terminate"): - await self.execute_code.terminate() - return rsp async def _act_on_task(self, current_task: Task) -> TaskResult: From bcde5171e0ccf86f8a51d4c7bb28ef18093fe255 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Mon, 11 Mar 2024 13:57:25 +0800 Subject: [PATCH 16/24] refine parse_outputs in ExecuteNbCode. --- metagpt/actions/mi/execute_nb_code.py | 31 ++++++++++--------- .../actions/mi/test_execute_nb_code.py | 17 ++++++++++ 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index 3a64a8bec1..4644ef5d5b 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -9,7 +9,6 @@ import asyncio import base64 import re -import traceback from typing import Literal, Tuple import nbformat @@ -92,17 +91,17 @@ def add_output_to_cell(self, cell: NotebookNode, output: str): else: cell["outputs"].append(new_output(output_type="stream", name="stdout", text=str(output))) - def parse_outputs(self, outputs: list[str]) -> str: + def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, str]: """Parses the outputs received from notebook execution.""" assert isinstance(outputs, list) - parsed_output = "" - + parsed_output, is_success = [], True for i, output in enumerate(outputs): + is_success = "traceback" not in output.keys() if output["output_type"] == "stream" and not any( tag in output["text"] for tag in ["| INFO | metagpt", "| ERROR | metagpt", "| WARNING | metagpt", "DEBUG"] ): - parsed_output += output["text"] + ioutput, is_success = truncate(remove_escape_and_color_codes(output["text"]), keep_len, is_success) elif output["output_type"] == "display_data": if "image/png" in output["data"]: self.show_bytes_figure(output["data"]["image/png"], self.interaction) @@ -110,9 +109,15 @@ def parse_outputs(self, outputs: list[str]) -> str: logger.info( f"{i}th output['data'] from nbclient outputs dont have image/png, continue next output ..." ) + ioutput, is_success = "", True elif output["output_type"] == "execute_result": - parsed_output += output["data"]["text/plain"] - return parsed_output + no_escape_color_output = remove_escape_and_color_codes(output["data"]["text/plain"]) + ioutput, is_success = truncate(no_escape_color_output, keep_len, is_success) + elif output["output_type"] == "error": + no_escape_color_output = remove_escape_and_color_codes("\n".join(output["traceback"])) + ioutput, is_success = truncate(no_escape_color_output, keep_len, is_success) + parsed_output.append(ioutput) + return is_success, ",".join(parsed_output) def show_bytes_figure(self, image_base64: str, interaction_type: Literal["ipython", None]): image_bytes = base64.b64decode(image_base64) @@ -157,7 +162,7 @@ async def run_cell(self, cell: NotebookNode, cell_index: int) -> Tuple[bool, str await self.reset() return False, "DeadKernelError" except Exception: - return False, f"{traceback.format_exc()}" + return False, "" async def run(self, code: str, language: Literal["python", "markdown"] = "python") -> Tuple[str, bool]: """ @@ -175,13 +180,9 @@ async def run(self, code: str, language: Literal["python", "markdown"] = "python # run code cell_index = len(self.nb.cells) - 1 success, error_message = await self.run_cell(self.nb.cells[-1], cell_index) - - if not success: - return truncate(remove_escape_and_color_codes(error_message), is_success=success) - - # code success - outputs = self.parse_outputs(self.nb.cells[-1].outputs) - outputs, success = truncate(remove_escape_and_color_codes(outputs), is_success=success) + success, outputs = self.parse_outputs(self.nb.cells[-1].outputs) + if error_message: + outputs = error_message + outputs if "!pip" in code: success = False diff --git a/tests/metagpt/actions/mi/test_execute_nb_code.py b/tests/metagpt/actions/mi/test_execute_nb_code.py index 3059ad3ae1..98c2e5cc31 100644 --- a/tests/metagpt/actions/mi/test_execute_nb_code.py +++ b/tests/metagpt/actions/mi/test_execute_nb_code.py @@ -100,6 +100,7 @@ async def test_terminate(): is_kernel_alive = await executor.nb_client.km.is_alive() assert is_kernel_alive await executor.terminate() + import time time.sleep(2) @@ -123,3 +124,19 @@ async def test_reset(): assert is_kernel_alive await executor.reset() assert executor.nb_client.km is None + + +@pytest.mark.asyncio +async def test_parse_outputs(): + executor = ExecuteNbCode() + code = """ + import pandas as pd + df = pd.DataFrame({'ID': [1,2,3], 'NAME': ['a', 'b', 'c']}) + print(df.columns) + print(df['DUMMPY_ID']) + """ + output, is_success = await executor.run(code) + assert not is_success + assert "Index(['ID', 'NAME'], dtype='object')" in output + assert "Executed code failed," in output + assert "KeyError: 'DUMMPY_ID'" in output From 9db705f20f79de8d77192bbd8adbc7dc37b25174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Mon, 11 Mar 2024 14:43:00 +0800 Subject: [PATCH 17/24] refine: rm truncate. --- metagpt/actions/mi/execute_nb_code.py | 38 +++++++++---------- .../actions/mi/test_execute_nb_code.py | 17 +-------- 2 files changed, 18 insertions(+), 37 deletions(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index 4644ef5d5b..2e22a7d0cd 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -101,7 +101,7 @@ def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, tag in output["text"] for tag in ["| INFO | metagpt", "| ERROR | metagpt", "| WARNING | metagpt", "DEBUG"] ): - ioutput, is_success = truncate(remove_escape_and_color_codes(output["text"]), keep_len, is_success) + ioutput, is_success = remove_escape_and_color_codes(output["text"]), True elif output["output_type"] == "display_data": if "image/png" in output["data"]: self.show_bytes_figure(output["data"]["image/png"], self.interaction) @@ -112,10 +112,24 @@ def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, ioutput, is_success = "", True elif output["output_type"] == "execute_result": no_escape_color_output = remove_escape_and_color_codes(output["data"]["text/plain"]) - ioutput, is_success = truncate(no_escape_color_output, keep_len, is_success) + ioutput, is_success = no_escape_color_output, True elif output["output_type"] == "error": no_escape_color_output = remove_escape_and_color_codes("\n".join(output["traceback"])) - ioutput, is_success = truncate(no_escape_color_output, keep_len, is_success) + ioutput, is_success = no_escape_color_output, False + + # handle coroutines that are not executed asynchronously + if ioutput.strip().startswith(" keep_len and is_success: + prefix = f"Executed code successfully. Truncated to show only first {keep_len} characters\n" + ioutput = prefix + ioutput[:keep_len] + elif len(ioutput) > keep_len and not is_success: + prefix = f"Executed code failed, please reflect the cause of bug and then debug. Truncated to show only last {keep_len} characters\n" + ioutput = prefix + ioutput[-keep_len:] + parsed_output.append(ioutput) return is_success, ",".join(parsed_output) @@ -198,24 +212,6 @@ async def run(self, code: str, language: Literal["python", "markdown"] = "python raise ValueError(f"Only support for language: python, markdown, but got {language}, ") -def truncate(result: str, keep_len: int = 2000, is_success: bool = True): - """对于超出keep_len个字符的result: 执行失败的代码, 展示result后keep_len个字符; 执行成功的代码, 展示result前keep_len个字符。""" - if is_success: - desc = f"Executed code successfully. Truncated to show only first {keep_len} characters\n" - else: - desc = f"Executed code failed, please reflect the cause of bug and then debug. Truncated to show only last {keep_len} characters\n" - - if result.strip().startswith(" keep_len: - result = result[-keep_len:] if not is_success else result[:keep_len] - return desc + result, is_success - - return result, is_success - - def remove_escape_and_color_codes(input_str: str): # 使用正则表达式去除jupyter notebook输出结果中的转义字符和颜色代码 # Use regular expressions to get rid of escape characters and color codes in jupyter notebook output. diff --git a/tests/metagpt/actions/mi/test_execute_nb_code.py b/tests/metagpt/actions/mi/test_execute_nb_code.py index 98c2e5cc31..4b90289ea9 100644 --- a/tests/metagpt/actions/mi/test_execute_nb_code.py +++ b/tests/metagpt/actions/mi/test_execute_nb_code.py @@ -1,6 +1,6 @@ import pytest -from metagpt.actions.mi.execute_nb_code import ExecuteNbCode, truncate +from metagpt.actions.mi.execute_nb_code import ExecuteNbCode @pytest.mark.asyncio @@ -54,21 +54,6 @@ async def test_plotting_code(): assert is_success -def test_truncate(): - # 代码执行成功 - output, is_success = truncate("hello world", 5, True) - assert "Truncated to show only first 5 characters\nhello" in output - assert is_success - # 代码执行失败 - output, is_success = truncate("hello world", 5, False) - assert "Truncated to show only last 5 characters\nworld" in output - assert not is_success - # 异步 - output, is_success = truncate(" Date: Mon, 11 Mar 2024 14:51:02 +0800 Subject: [PATCH 18/24] refine: rm `is_success = "traceback" not in output.keys()` --- metagpt/actions/mi/execute_nb_code.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index 2e22a7d0cd..632f0076c2 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -96,7 +96,6 @@ def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, assert isinstance(outputs, list) parsed_output, is_success = [], True for i, output in enumerate(outputs): - is_success = "traceback" not in output.keys() if output["output_type"] == "stream" and not any( tag in output["text"] for tag in ["| INFO | metagpt", "| ERROR | metagpt", "| WARNING | metagpt", "DEBUG"] From cc00161b989eed48c94c4425a89279f534ed942d Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Mon, 11 Mar 2024 16:25:02 +0800 Subject: [PATCH 19/24] simplify save_webpages in gpt_v_generator.py --- metagpt/tools/libs/gpt_v_generator.py | 61 +++++++------------ .../tools/libs/test_gpt_v_generator.py | 4 +- 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/metagpt/tools/libs/gpt_v_generator.py b/metagpt/tools/libs/gpt_v_generator.py index 3b17fc596f..bcca0df7b9 100644 --- a/metagpt/tools/libs/gpt_v_generator.py +++ b/metagpt/tools/libs/gpt_v_generator.py @@ -6,12 +6,13 @@ @File : gpt_v_generator.py """ import os +import re from pathlib import Path from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.tools.tool_registry import register_tool -from metagpt.tools.tool_type import ToolType -from metagpt.utils.common import encode_image +from metagpt.utils.common import encode_image, CodeParser +from metagpt.logs import logger ANALYZE_LAYOUT_PROMPT = """You are now a UI/UX designer, please generate layout information for this image: @@ -28,11 +29,9 @@ Now, please generate the corresponding webpage code including HTML, CSS and JavaScript:""" -@register_tool( - tool_type=ToolType.IMAGE2WEBPAGE.type_name, include_functions=["__init__", "generate_webpages", "save_webpages"] -) +@register_tool(include_functions=["__init__", "generate_webpages", "save_webpages"]) class GPTvGenerator: - """Class for generating webpages at once. + """Class for generating webpage code from a given webpage screenshot. This class provides methods to generate webpages including all code (HTML, CSS, and JavaScript) based on an image. It utilizes a vision model to analyze the layout from an image and generate webpage codes accordingly. @@ -75,50 +74,34 @@ async def generate_webpages(self, image_path: str) -> str: return await self.llm.aask(msg=prompt, images=[encode_image(image_path)]) @staticmethod - def save_webpages(image_path: str, webpages: str) -> Path: + def save_webpages(webpages: str, save_folder_name: str = "example") -> Path: """Save webpages including all code (HTML, CSS, and JavaScript) at once. Args: - image_path (str): The path of the image file. webpages (str): The generated webpages content. + save_folder_name (str, optional): The name of the folder to save the webpages. Defaults to 'example'. Returns: Path: The path of the saved webpages. """ # Create a folder called webpages in the workspace directory to store HTML, CSS, and JavaScript files - webpages_path = DEFAULT_WORKSPACE_ROOT / "webpages" / Path(image_path).stem + webpages_path = DEFAULT_WORKSPACE_ROOT / "webpages" / save_folder_name + logger.info(f"code will be saved at {webpages_path}") os.makedirs(webpages_path, exist_ok=True) index_path = webpages_path / "index.html" - try: - index = webpages.split("```html")[1].split("```")[0] - style_path = None - if "styles.css" in index: - style_path = webpages_path / "styles.css" - elif "style.css" in index: - style_path = webpages_path / "style.css" - style = webpages.split("```css")[1].split("```")[0] if style_path else "" - - js_path = None - if "scripts.js" in index: - js_path = webpages_path / "scripts.js" - elif "script.js" in index: - js_path = webpages_path / "script.js" - - js = webpages.split("```javascript")[1].split("```")[0] if js_path else "" - except IndexError: - raise ValueError(f"No html or css or js code found in the result. \nWebpages: {webpages}") - - try: - with open(index_path, "w", encoding="utf-8") as f: - f.write(index) - if style_path: - with open(style_path, "w", encoding="utf-8") as f: - f.write(style) - if js_path: - with open(js_path, "w", encoding="utf-8") as f: - f.write(js) - except FileNotFoundError as e: - raise FileNotFoundError(f"Cannot save the webpages to {str(webpages_path)}") from e + index_path.write_text(CodeParser.parse_code(block=None, text=webpages, lang="html")) + + extract_and_save_code(folder=webpages_path, text=webpages, pattern="styles?.css", language="css") + + extract_and_save_code(folder=webpages_path, text=webpages, pattern="scripts?.js", language="javascript") return webpages_path + + +def extract_and_save_code(folder, text, pattern, language): + word = re.search(pattern, text) + if word: + path = folder / word.group(0) + code = CodeParser.parse_code(block=None, text=text, lang=language) + path.write_text(code, encoding="utf-8") diff --git a/tests/metagpt/tools/libs/test_gpt_v_generator.py b/tests/metagpt/tools/libs/test_gpt_v_generator.py index 907006765f..96b61ccca6 100644 --- a/tests/metagpt/tools/libs/test_gpt_v_generator.py +++ b/tests/metagpt/tools/libs/test_gpt_v_generator.py @@ -60,7 +60,7 @@ async def test_generate_webpages(mock_webpage_filename_with_styles_and_scripts, async def test_save_webpages_with_styles_and_scripts(mock_webpage_filename_with_styles_and_scripts, image_path): generator = GPTvGenerator() webpages = await generator.generate_webpages(image_path) - webpages_dir = generator.save_webpages(image_path=image_path, webpages=webpages) + webpages_dir = generator.save_webpages(webpages=webpages, save_folder_name="test_1") logs.logger.info(webpages_dir) assert webpages_dir.exists() @@ -69,7 +69,7 @@ async def test_save_webpages_with_styles_and_scripts(mock_webpage_filename_with_ async def test_save_webpages_with_style_and_script(mock_webpage_filename_with_style_and_script, image_path): generator = GPTvGenerator() webpages = await generator.generate_webpages(image_path) - webpages_dir = generator.save_webpages(image_path=image_path, webpages=webpages) + webpages_dir = generator.save_webpages(webpages=webpages, save_folder_name="test_2") logs.logger.info(webpages_dir) assert webpages_dir.exists() From 478ba13b5f1e25948d39c634b98bc9c188e80b8c Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Mon, 11 Mar 2024 16:56:47 +0800 Subject: [PATCH 20/24] simplify save_webpages in gpt_v_generator.py --- metagpt/tools/libs/gpt_v_generator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/tools/libs/gpt_v_generator.py b/metagpt/tools/libs/gpt_v_generator.py index bcca0df7b9..f24a473bfb 100644 --- a/metagpt/tools/libs/gpt_v_generator.py +++ b/metagpt/tools/libs/gpt_v_generator.py @@ -5,7 +5,6 @@ @Author : mannaandpoem @File : gpt_v_generator.py """ -import os import re from pathlib import Path @@ -87,7 +86,7 @@ def save_webpages(webpages: str, save_folder_name: str = "example") -> Path: # Create a folder called webpages in the workspace directory to store HTML, CSS, and JavaScript files webpages_path = DEFAULT_WORKSPACE_ROOT / "webpages" / save_folder_name logger.info(f"code will be saved at {webpages_path}") - os.makedirs(webpages_path, exist_ok=True) + webpages_path.mkdir(parents=True, exist_ok=True) index_path = webpages_path / "index.html" index_path.write_text(CodeParser.parse_code(block=None, text=webpages, lang="html")) From 09e3a8e1fa98a6d03507f6f4db9ba2a0064d15fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Mon, 11 Mar 2024 17:00:44 +0800 Subject: [PATCH 21/24] refine run_cell and parse_outputs. --- metagpt/actions/mi/execute_nb_code.py | 36 ++++++++----------- .../actions/mi/test_execute_nb_code.py | 5 +-- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index 632f0076c2..217fc8ddcb 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -96,11 +96,12 @@ def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, assert isinstance(outputs, list) parsed_output, is_success = [], True for i, output in enumerate(outputs): + output_text = "" if output["output_type"] == "stream" and not any( tag in output["text"] for tag in ["| INFO | metagpt", "| ERROR | metagpt", "| WARNING | metagpt", "DEBUG"] ): - ioutput, is_success = remove_escape_and_color_codes(output["text"]), True + output_text = output["text"] elif output["output_type"] == "display_data": if "image/png" in output["data"]: self.show_bytes_figure(output["data"]["image/png"], self.interaction) @@ -108,28 +109,22 @@ def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, logger.info( f"{i}th output['data'] from nbclient outputs dont have image/png, continue next output ..." ) - ioutput, is_success = "", True elif output["output_type"] == "execute_result": - no_escape_color_output = remove_escape_and_color_codes(output["data"]["text/plain"]) - ioutput, is_success = no_escape_color_output, True + output_text = output["data"]["text/plain"] elif output["output_type"] == "error": - no_escape_color_output = remove_escape_and_color_codes("\n".join(output["traceback"])) - ioutput, is_success = no_escape_color_output, False + output_text, is_success = "\n".join(output["traceback"]), False # handle coroutines that are not executed asynchronously - if ioutput.strip().startswith(" keep_len and is_success: - prefix = f"Executed code successfully. Truncated to show only first {keep_len} characters\n" - ioutput = prefix + ioutput[:keep_len] - elif len(ioutput) > keep_len and not is_success: - prefix = f"Executed code failed, please reflect the cause of bug and then debug. Truncated to show only last {keep_len} characters\n" - ioutput = prefix + ioutput[-keep_len:] + output_text = remove_escape_and_color_codes(output_text) + # The valid information of the exception is at the end, + # the valid information of Normal output is at the begining. + output_text = output_text[:keep_len] if is_success else output_text[-keep_len:] - parsed_output.append(ioutput) + parsed_output.append(output_text) return is_success, ",".join(parsed_output) def show_bytes_figure(self, image_base64: str, interaction_type: Literal["ipython", None]): @@ -164,7 +159,7 @@ async def run_cell(self, cell: NotebookNode, cell_index: int) -> Tuple[bool, str """ try: await self.nb_client.async_execute_cell(cell, cell_index) - return True, "" + return self.parse_outputs(self.nb.cells[-1].outputs) except CellTimeoutError: assert self.nb_client.km is not None await self.nb_client.km.interrupt_kernel() @@ -175,7 +170,7 @@ async def run_cell(self, cell: NotebookNode, cell_index: int) -> Tuple[bool, str await self.reset() return False, "DeadKernelError" except Exception: - return False, "" + return self.parse_outputs(self.nb.cells[-1].outputs) async def run(self, code: str, language: Literal["python", "markdown"] = "python") -> Tuple[str, bool]: """ @@ -192,10 +187,7 @@ async def run(self, code: str, language: Literal["python", "markdown"] = "python # run code cell_index = len(self.nb.cells) - 1 - success, error_message = await self.run_cell(self.nb.cells[-1], cell_index) - success, outputs = self.parse_outputs(self.nb.cells[-1].outputs) - if error_message: - outputs = error_message + outputs + success, outputs = await self.run_cell(self.nb.cells[-1], cell_index) if "!pip" in code: success = False diff --git a/tests/metagpt/actions/mi/test_execute_nb_code.py b/tests/metagpt/actions/mi/test_execute_nb_code.py index 4b90289ea9..2ecfbd2a2d 100644 --- a/tests/metagpt/actions/mi/test_execute_nb_code.py +++ b/tests/metagpt/actions/mi/test_execute_nb_code.py @@ -68,7 +68,7 @@ async def test_run_code_text(): executor = ExecuteNbCode() message, success = await executor.run(code='print("This is a code!")', language="python") assert success - assert message == "This is a code!\n" + assert "This is a code!" in message message, success = await executor.run(code="# This is a code!", language="markdown") assert success assert message == "# This is a code!" @@ -118,10 +118,11 @@ async def test_parse_outputs(): import pandas as pd df = pd.DataFrame({'ID': [1,2,3], 'NAME': ['a', 'b', 'c']}) print(df.columns) + print(f"columns num:{len(df.columns)}") print(df['DUMMPY_ID']) """ output, is_success = await executor.run(code) assert not is_success assert "Index(['ID', 'NAME'], dtype='object')" in output - assert "Executed code failed," in output assert "KeyError: 'DUMMPY_ID'" in output + assert "columns num:2" in output From 980851136fb1e63dc6bd04c33046f68f167b4b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Mon, 11 Mar 2024 17:07:54 +0800 Subject: [PATCH 22/24] chore --- metagpt/actions/mi/execute_nb_code.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/mi/execute_nb_code.py b/metagpt/actions/mi/execute_nb_code.py index 217fc8ddcb..f6a8defbde 100644 --- a/metagpt/actions/mi/execute_nb_code.py +++ b/metagpt/actions/mi/execute_nb_code.py @@ -120,8 +120,8 @@ def parse_outputs(self, outputs: list[str], keep_len: int = 2000) -> Tuple[bool, is_success = False output_text = remove_escape_and_color_codes(output_text) - # The valid information of the exception is at the end, - # the valid information of Normal output is at the begining. + # The useful information of the exception is at the end, + # the useful information of normal output is at the begining. output_text = output_text[:keep_len] if is_success else output_text[-keep_len:] parsed_output.append(output_text) From 363c1b7ffb0fe9f497dfc83e59cc711067bbf36a Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Mon, 11 Mar 2024 18:15:02 +0800 Subject: [PATCH 23/24] simplify save_webpages in gpt_v_generator.py --- tests/metagpt/tools/libs/test_gpt_v_generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/metagpt/tools/libs/test_gpt_v_generator.py b/tests/metagpt/tools/libs/test_gpt_v_generator.py index 96b61ccca6..4a2e686825 100644 --- a/tests/metagpt/tools/libs/test_gpt_v_generator.py +++ b/tests/metagpt/tools/libs/test_gpt_v_generator.py @@ -63,6 +63,9 @@ async def test_save_webpages_with_styles_and_scripts(mock_webpage_filename_with_ webpages_dir = generator.save_webpages(webpages=webpages, save_folder_name="test_1") logs.logger.info(webpages_dir) assert webpages_dir.exists() + assert (webpages_dir / "index.html").exists() + assert (webpages_dir / "styles.css").exists() + assert (webpages_dir / "scripts.js").exists() @pytest.mark.asyncio @@ -72,6 +75,9 @@ async def test_save_webpages_with_style_and_script(mock_webpage_filename_with_st webpages_dir = generator.save_webpages(webpages=webpages, save_folder_name="test_2") logs.logger.info(webpages_dir) assert webpages_dir.exists() + assert (webpages_dir / "index.html").exists() + assert (webpages_dir / "style.css").exists() + assert (webpages_dir / "script.js").exists() @pytest.mark.asyncio From 9ad2f22c16247a159186a90c3335125acacd41da Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Mon, 11 Mar 2024 19:59:12 +0800 Subject: [PATCH 24/24] simplify save_webpages in gpt_v_generator.py --- metagpt/tools/libs/gpt_v_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/tools/libs/gpt_v_generator.py b/metagpt/tools/libs/gpt_v_generator.py index f24a473bfb..0e9f347703 100644 --- a/metagpt/tools/libs/gpt_v_generator.py +++ b/metagpt/tools/libs/gpt_v_generator.py @@ -9,9 +9,9 @@ from pathlib import Path from metagpt.const import DEFAULT_WORKSPACE_ROOT -from metagpt.tools.tool_registry import register_tool -from metagpt.utils.common import encode_image, CodeParser from metagpt.logs import logger +from metagpt.tools.tool_registry import register_tool +from metagpt.utils.common import CodeParser, encode_image ANALYZE_LAYOUT_PROMPT = """You are now a UI/UX designer, please generate layout information for this image: