From a6c58818f61267b626e8e7a279578eb3b40bb091 Mon Sep 17 00:00:00 2001 From: Eivind Jahren Date: Thu, 7 Sep 2023 13:39:19 +0200 Subject: [PATCH] Fix gen_data_rft_export --- src/ert/gui/tools/plugins/plugin_runner.py | 1 + .../gui/tools/plugins/process_job_dialog.py | 14 +- .../scripts/gen_data_rft_export.py | 58 +++++---- tests/unit_tests/gui/conftest.py | 44 ++++++- tests/unit_tests/gui/test_main_window.py | 43 +----- .../unit_tests/gui/test_rft_export_plugin.py | 122 ++++++++++++++++++ 6 files changed, 211 insertions(+), 71 deletions(-) create mode 100644 tests/unit_tests/gui/test_rft_export_plugin.py diff --git a/src/ert/gui/tools/plugins/plugin_runner.py b/src/ert/gui/tools/plugins/plugin_runner.py index 80fbbe0008c..58450e5c1e7 100644 --- a/src/ert/gui/tools/plugins/plugin_runner.py +++ b/src/ert/gui/tools/plugins/plugin_runner.py @@ -29,6 +29,7 @@ def run(self): arguments = plugin.getArguments() dialog = ProcessJobDialog(plugin.getName(), plugin.getParentWindow()) + dialog.setObjectName("process_job_dialog") dialog.cancelConfirmed.connect(self.cancel) diff --git a/src/ert/gui/tools/plugins/process_job_dialog.py b/src/ert/gui/tools/plugins/process_job_dialog.py index 16055051ea2..b7d66ebc3c0 100644 --- a/src/ert/gui/tools/plugins/process_job_dialog.py +++ b/src/ert/gui/tools/plugins/process_job_dialog.py @@ -73,6 +73,8 @@ def __init__(self, title, parent=None): self.presentError.connect(self.__presentError) self.closeButtonPressed.connect(self.__confirmCancel) + self._msg_box = None + def disableCloseButton(self): self.close_button.setEnabled(False) @@ -106,16 +108,16 @@ def __createMsgBox(self, title, message, details): return msg_box def __presentInformation(self, title, message, details): - msg_box = self.__createMsgBox(title, message, details) - msg_box.setIcon(QMessageBox.Information) + self._msg_box = self.__createMsgBox(title, message, details) + self._msg_box.setIcon(QMessageBox.Information) - msg_box.exec_() + self._msg_box.exec_() def __presentError(self, title, message, details): - msg_box = self.__createMsgBox(title, message, details) - msg_box.setIcon(QMessageBox.Critical) + self._msg_box = self.__createMsgBox(title, message, details) + self._msg_box.setIcon(QMessageBox.Critical) - msg_box.exec_() + self._msg_box.exec_() def __confirmCancel(self): cancel_box = self.__createMsgBox( diff --git a/src/ert/shared/share/ert/workflows/jobs/internal-gui/scripts/gen_data_rft_export.py b/src/ert/shared/share/ert/workflows/jobs/internal-gui/scripts/gen_data_rft_export.py index 28ac6ab4196..14c609ddf35 100644 --- a/src/ert/shared/share/ert/workflows/jobs/internal-gui/scripts/gen_data_rft_export.py +++ b/src/ert/shared/share/ert/workflows/jobs/internal-gui/scripts/gen_data_rft_export.py @@ -3,7 +3,7 @@ import re import numpy -import pandas +import pandas as pd from ert import LibresFacade from ert.config import ( @@ -44,7 +44,6 @@ def load_args(filename, column_names=None): with open(filename, encoding="utf-8") as fileH: for line in fileH.readlines(): tmp = line.split() - print(tmp) for column in range(columns): data[row][column] = float(tmp[column]) row += 1 @@ -54,7 +53,7 @@ def load_args(filename, column_names=None): for column in range(columns): column_names.append(f"Column{column:d}") - data_frame = pandas.DataFrame(data=data, columns=column_names) + data_frame = pd.DataFrame(data=data, columns=column_names) return data_frame @@ -144,36 +143,46 @@ def run( # pylint: disable=arguments-differ if case_list is not None: cases = case_list.split(",") - if case_list is None or len(cases) == 0: - cases = ["default"] + if len(cases) == 0: + raise UserWarning("No cases given to load from") - data_frame = pandas.DataFrame() + data = [] for case in cases: case = case.strip() - case_frame = pandas.DataFrame() + case_data = [] try: ensemble = self.storage.get_ensemble_by_name(case) except KeyError as exc: raise UserWarning(f"The case '{case}' does not exist!") from exc - if not ensemble.has_data(): + if not ensemble.has_data: raise UserWarning(f"The case '{case}' does not have any data!") + if len(obs_keys) == 0: + raise UserWarning( + "The config does not contain any" + " GENERAL_OBSERVATIONS starting with RFT_*" + ) + for obs_key in obs_keys: well = obs_key.replace("RFT_", "") wells.add(well) obs_vector = enkf_obs[obs_key] data_key = obs_vector.data_key - report_step = obs_vector.activeStep() - obs_node = obs_vector.observations[report_step] + if len(obs_vector.observations) == 1: + report_step, obs_node = list(obs_vector.observations.items())[0] + else: + raise UserWarning( + "GEN_DATA RFT CSV Export can only be used for observations " + "active for exactly one report step" + ) - rft_data = facade.load_gen_data(case, data_key, report_step) - fs = self.storage.get_ensemble_by_name(case) - realizations = fs.realization_list(RealizationState.HAS_DATA) + rft_data = facade.load_gen_data(ensemble, data_key, report_step) + realizations = ensemble.realization_list(RealizationState.HAS_DATA) # Trajectory - trajectory_file = os.path.join(trajectory_path, f"{well}.txt" % well) + trajectory_file = os.path.join(trajectory_path, f"{well}.txt") if not os.path.isfile(trajectory_file): trajectory_file = os.path.join(trajectory_path, f"{well}_R.txt") @@ -187,14 +196,14 @@ def run( # pylint: disable=arguments-differ obs = numpy.empty(shape=(data_size, 2), dtype=numpy.float64) obs.fill(numpy.nan) for obs_index in range(len(obs_node)): - data_index = obs_node.getDataIndex(obs_index) - value = obs_node.getValue(obs_index) - std = obs_node.getStandardDeviation(obs_index) + data_index = obs_node.indices[obs_index] + value = obs_node.values[obs_index] + std = obs_node.stds[obs_index] obs[data_index, 0] = value obs[data_index, 1] = std for iens in realizations: - realization_frame = pandas.DataFrame( + realization_frame = pd.DataFrame( data={ "TVD": tvd_arg, "Pressure": rft_data[iens], @@ -209,15 +218,16 @@ def run( # pylint: disable=arguments-differ realization_frame["Case"] = case realization_frame["Iteration"] = ensemble.iteration - case_frame = case_frame.append(realization_frame) + case_data.append(realization_frame) - data_frame = data_frame.append(case_frame) + data.append(pd.concat(case_data)) - data_frame.set_index(["Realization", "Well", "Case", "Iteration"], inplace=True) + frame = pd.concat(data) + frame.set_index(["Realization", "Well", "Case", "Iteration"], inplace=True) if drop_const_cols: - data_frame = data_frame.loc[:, (data_frame != data_frame.iloc[0]).any()] + frame = frame.loc[:, (frame != frame.iloc[0]).any()] - data_frame.to_csv(output_file) + frame.to_csv(output_file) well_list_str = ", ".join(list(wells)) export_info = ( f"Exported RFT information for wells: {well_list_str} to: {output_file}" @@ -237,9 +247,11 @@ def getArguments(self, parent=None): "wellpath", must_be_a_directory=True, must_be_a_file=False, must_exist=True ) trajectory_chooser = PathChooser(trajectory_model) + trajectory_chooser.setObjectName("trajectory_chooser") all_case_list = [case.name for case in self.storage.ensembles] list_edit = ListEditBox(all_case_list) + list_edit.setObjectName("list_of_cases") infer_iteration_check = QCheckBox() infer_iteration_check.setChecked(True) diff --git a/tests/unit_tests/gui/conftest.py b/tests/unit_tests/gui/conftest.py index 7cbc65519ec..7d76c066bf8 100644 --- a/tests/unit_tests/gui/conftest.py +++ b/tests/unit_tests/gui/conftest.py @@ -1,6 +1,7 @@ import contextlib import copy import fileinput +import os import os.path import shutil import stat @@ -13,7 +14,7 @@ import pytest from pytestqt.qtbot import QtBot from qtpy.QtCore import Qt, QTimer -from qtpy.QtWidgets import QComboBox, QMessageBox, QWidget +from qtpy.QtWidgets import QApplication, QComboBox, QMessageBox, QPushButton, QWidget from ert.config import ErtConfig from ert.enkf_main import EnKFMain @@ -31,13 +32,15 @@ REALIZATION_STATE_UNKNOWN, STEP_STATE_UNKNOWN, ) +from ert.gui.ertwidgets import ClosableDialog from ert.gui.ertwidgets.caselist import AddRemoveWidget, CaseList -from ert.gui.ertwidgets.closabledialog import ClosableDialog +from ert.gui.ertwidgets.caseselector import CaseSelector from ert.gui.ertwidgets.validateddialog import ValidatedDialog from ert.gui.main import GUILogHandler, _setup_main_window from ert.gui.simulation.run_dialog import RunDialog from ert.gui.simulation.simulation_panel import SimulationPanel from ert.gui.simulation.view import RealizationWidget +from ert.gui.tools.load_results.load_results_panel import LoadResultsPanel from ert.gui.tools.manage_cases.case_init_configuration import ( CaseInitializationConfigurationPanel, ) @@ -390,3 +393,40 @@ def _make_mock_tracker(events): return MockTracker(events) return _make_mock_tracker + + +def load_results_manually(qtbot, gui, case_name="default"): + def handle_load_results_dialog(): + qtbot.waitUntil( + lambda: gui.findChild(ClosableDialog, name="load_results_manually_tool") + is not None + ) + dialog = gui.findChild(ClosableDialog, name="load_results_manually_tool") + panel = dialog.findChild(LoadResultsPanel) + assert isinstance(panel, LoadResultsPanel) + + case_selector = panel.findChild(CaseSelector) + assert isinstance(case_selector, CaseSelector) + index = case_selector.findText(case_name, Qt.MatchFlag.MatchContains) + assert index != -1 + case_selector.setCurrentIndex(index) + + # click on "Load" + load_button = panel.parent().findChild(QPushButton, name="Load") + assert isinstance(load_button, QPushButton) + + # Verify that the messagebox is the success kind + def handle_popup_dialog(): + messagebox = QApplication.activeModalWidget() + assert isinstance(messagebox, QMessageBox) + assert messagebox.text() == "Successfully loaded all realisations" + ok_button = messagebox.button(QMessageBox.Ok) + qtbot.mouseClick(ok_button, Qt.LeftButton) + + QTimer.singleShot(1000, handle_popup_dialog) + qtbot.mouseClick(load_button, Qt.LeftButton) + dialog.close() + + QTimer.singleShot(1000, handle_load_results_dialog) + load_results_tool = gui.tools["Load results manually"] + load_results_tool.trigger() diff --git a/tests/unit_tests/gui/test_main_window.py b/tests/unit_tests/gui/test_main_window.py index 7572d0e0594..9926870bd6f 100644 --- a/tests/unit_tests/gui/test_main_window.py +++ b/tests/unit_tests/gui/test_main_window.py @@ -8,7 +8,6 @@ import pytest from qtpy.QtCore import Qt, QTimer from qtpy.QtWidgets import ( - QApplication, QComboBox, QMessageBox, QPushButton, @@ -27,7 +26,6 @@ from ert.gui.ertwidgets.validateddialog import ValidatedDialog from ert.gui.simulation.run_dialog import RunDialog from ert.gui.simulation.simulation_panel import SimulationPanel -from ert.gui.tools.load_results.load_results_panel import LoadResultsPanel from ert.gui.tools.manage_cases.case_init_configuration import ( CaseInitializationConfigurationPanel, ) @@ -36,7 +34,7 @@ from ert.gui.tools.plot.plot_window import PlotWindow from ert.run_models import SingleTestRun -from .conftest import find_cases_dialog_and_panel +from .conftest import find_cases_dialog_and_panel, load_results_manually @pytest.mark.usefixtures("use_tmpdir") @@ -347,45 +345,10 @@ def handle_add_dialog(): @pytest.mark.usefixtures("use_tmpdir") -def test_that_the_load_results_manually_tool_works( +def test_that_load_results_manually_can_be_run_after_esmda( esmda_has_run, opened_main_window, qtbot ): - gui = opened_main_window - - def handle_load_results_dialog(): - qtbot.waitUntil( - lambda: gui.findChild(ClosableDialog, name="load_results_manually_tool") - is not None - ) - dialog = gui.findChild(ClosableDialog, name="load_results_manually_tool") - panel = dialog.findChild(LoadResultsPanel) - assert isinstance(panel, LoadResultsPanel) - - case_selector = panel.findChild(CaseSelector) - assert isinstance(case_selector, CaseSelector) - index = case_selector.findText("default", Qt.MatchFlag.MatchContains) - assert index != -1 - case_selector.setCurrentIndex(index) - - # click on "Load" - load_button = panel.parent().findChild(QPushButton, name="Load") - assert isinstance(load_button, QPushButton) - - # Verify that the messagebox is the success kind - def handle_popup_dialog(): - messagebox = QApplication.activeModalWidget() - assert isinstance(messagebox, QMessageBox) - assert messagebox.text() == "Successfully loaded all realisations" - ok_button = messagebox.button(QMessageBox.Ok) - qtbot.mouseClick(ok_button, Qt.LeftButton) - - QTimer.singleShot(1000, handle_popup_dialog) - qtbot.mouseClick(load_button, Qt.LeftButton) - dialog.close() - - QTimer.singleShot(1000, handle_load_results_dialog) - load_results_tool = gui.tools["Load results manually"] - load_results_tool.trigger() + load_results_manually(qtbot, opened_main_window) @pytest.mark.usefixtures("use_tmpdir") diff --git a/tests/unit_tests/gui/test_rft_export_plugin.py b/tests/unit_tests/gui/test_rft_export_plugin.py new file mode 100644 index 00000000000..7fb215e52e0 --- /dev/null +++ b/tests/unit_tests/gui/test_rft_export_plugin.py @@ -0,0 +1,122 @@ +import os +from pathlib import Path +from textwrap import dedent +from unittest.mock import Mock + +import pytest +from qtpy.QtCore import QTimer + +from ert.config import ErtConfig +from ert.enkf_main import EnKFMain +from ert.gui.ertwidgets.customdialog import CustomDialog +from ert.gui.ertwidgets.listeditbox import ListEditBox +from ert.gui.ertwidgets.pathchooser import PathChooser +from ert.gui.main import GUILogHandler, _setup_main_window +from ert.services import StorageService +from ert.storage import open_storage + +from .conftest import load_results_manually + + +@pytest.fixture +def well_file(tmp_path): + (tmp_path / "OBS.txt").write_text("1.0 2.0 3.0 4.0") + + +@pytest.fixture +def ert_rft_setup(tmp_path): + (tmp_path / "config.ert").write_text( + dedent( + """ + NUM_REALIZATIONS 3 + OBS_CONFIG obs + GEN_DATA RFT_DATA INPUT_FORMAT:ASCII RESULT_FILE:rft_%d.txt REPORT_STEPS:0 + TIME_MAP time_map.txt + """ + ) + ) + (tmp_path / "obs").write_text( + dedent( + """ + GENERAL_OBSERVATION RFT_OBS { + DATA = RFT_DATA; + DATE = 3001-09-01; -- The final odyssey + VALUE = 42.0; + ERROR = 0.69420; + }; + """ + ) + ) + (tmp_path / "time_map.txt").write_text("3001-09-01\n") + return tmp_path / "config.ert" + + +@pytest.fixture +def gen_data_in_runpath(tmp_path): + simulations_dir = tmp_path / "simulations" + simulations_dir.mkdir() + for i in range(3): + realization_dir = simulations_dir / f"realization-{i}" + realization_dir.mkdir() + (realization_dir / "iter-0").mkdir() + (realization_dir / "iter-0" / "rft_0.txt").write_text( + f"{i}.0", encoding="utf-8" + ) + + +@pytest.mark.usefixtures("use_tmpdir") +def test_rft_csv_export_plugin_exports_rft_data( + qtbot, ert_rft_setup, well_file, gen_data_in_runpath +): + args = Mock() + args.config = str(ert_rft_setup) + + output_file = Path("output.csv") + + ert_config = ErtConfig.from_file(args.config) + enkf_main = EnKFMain(ert_config) + with StorageService.init_service( + ert_config=args.config, + project=os.path.abspath(ert_config.ens_path), + ), open_storage(ert_config.ens_path, mode="w") as storage: + gui = _setup_main_window(enkf_main, args, GUILogHandler()) + qtbot.addWidget(gui) + gui.notifier.set_storage(storage) + gui.notifier.set_current_case( + storage.create_experiment( + parameters=enkf_main.ensembleConfig().parameter_configuration + ).create_ensemble( + name="default", + ensemble_size=enkf_main.getEnsembleSize(), + ) + ) + + load_results_manually(qtbot, gui) + + def handle_rft_plugin_dialog(): + qtbot.waitUntil(lambda: gui.findChild(CustomDialog) is not None) + dialog = gui.findChild(CustomDialog) + assert isinstance(dialog, CustomDialog) + trajectory_field = dialog.findChild(PathChooser, name="trajectory_chooser") + assert isinstance(trajectory_field, PathChooser) + trajectory_field._model.setValue(".") + list_field = dialog.findChild(ListEditBox, name="list_of_cases") + assert isinstance(list_field, ListEditBox) + list_field._list_edit_line.setText("default") + dialog.accept() + + QTimer.singleShot(1000, handle_rft_plugin_dialog) + plugin_actions = gui.tools["Plugins"].getAction().menu().actions() + rft_plugin = [ + a for a in plugin_actions if a.iconText() == "GEN_DATA RFT CSV Export" + ][0] + rft_plugin.trigger() + qtbot.waitUntil(output_file.exists, timeout=20000) + assert output_file.read_text(encoding="utf-8") == dedent( + """\ + Realization,Well,Case,Iteration,Pressure + 0,OBS,default,0,0.0 + 1,OBS,default,0,1.0 + 2,OBS,default,0,2.0 + """ + )