From 77e3139642d5b261764aa21f5aa4cf876e1bf7e7 Mon Sep 17 00:00:00 2001 From: Jon Holba Date: Thu, 14 Mar 2024 09:50:57 +0100 Subject: [PATCH] Rewrite gui tests to have a clean app/storage for each test --- tests/unit_tests/gui/conftest.py | 290 +++++++++++------- .../gui/test_full_manual_update_workflow.py | 42 +-- tests/unit_tests/gui/test_main_window.py | 39 +-- 3 files changed, 202 insertions(+), 169 deletions(-) diff --git a/tests/unit_tests/gui/conftest.py b/tests/unit_tests/gui/conftest.py index 53dcec9ccbd..04781a74598 100644 --- a/tests/unit_tests/gui/conftest.py +++ b/tests/unit_tests/gui/conftest.py @@ -6,9 +6,10 @@ import shutil import stat import time +from contextlib import contextmanager from datetime import datetime as dt from textwrap import dedent -from typing import List, Type, TypeVar +from typing import Generator, List, Tuple, Type, TypeVar from unittest.mock import MagicMock, Mock import pytest @@ -36,7 +37,6 @@ from ert.gui.ertwidgets.ensembleselector import EnsembleSelector from ert.gui.ertwidgets.storage_widget import StorageWidget from ert.gui.main import ErtMainWindow, GUILogHandler, _setup_main_window -from ert.gui.simulation.ensemble_experiment_panel import EnsembleExperimentPanel from ert.gui.simulation.run_dialog import RunDialog from ert.gui.simulation.simulation_panel import SimulationPanel from ert.gui.simulation.view import RealizationWidget @@ -46,7 +46,7 @@ ) from ert.run_models import EnsembleExperiment, MultipleDataAssimilation from ert.services import StorageService -from ert.storage import open_storage +from ert.storage import Storage, open_storage from tests.unit_tests.gui.simulation.test_run_path_dialog import handle_run_path_dialog @@ -61,85 +61,174 @@ def handle_manage_dialog(): manage_tool.trigger() -@pytest.fixture(name="opened_main_window", scope="module") -def opened_main_window_fixture(source_root, tmpdir_factory) -> ErtMainWindow: - with pytest.MonkeyPatch.context() as mp: - tmp_path = tmpdir_factory.mktemp("test-data") - shutil.copytree( - os.path.join(source_root, "test-data", "poly_example"), - tmp_path / "test_data", +@pytest.fixture +def opened_main_window( + source_root, tmp_path, monkeypatch +) -> Generator[ErtMainWindow, None, None]: + monkeypatch.chdir(tmp_path) + _new_poly_example(source_root, tmp_path) + with _open_main_window(tmp_path) as ( + gui, + storage, + config, + ), StorageService.init_service( + project=os.path.abspath(config.ens_path), + ): + _add_default_ensemble(storage, gui, config) + yield gui + + +def _new_poly_example(source_root, destination): + shutil.copytree( + os.path.join(source_root, "test-data", "poly_example"), + destination, + dirs_exist_ok=True, + ) + + with fileinput.input(destination / "poly.ert", inplace=True) as fin: + for line in fin: + if "NUM_REALIZATIONS" in line: + # Decrease the number of realizations to speed up the test, + # if there is flakyness, this can be increased. + print("NUM_REALIZATIONS 20", end="\n") + else: + print(line, end="") + + +def _add_default_ensemble(storage: Storage, gui: ErtMainWindow, config: ErtConfig): + gui.notifier.set_current_ensemble( + storage.create_experiment( + parameters=config.ensemble_config.parameter_configuration, + observations=config.observations, + ).create_ensemble( + name="default", + ensemble_size=config.model_config.num_realizations, ) - mp.chdir(tmp_path / "test_data") - with fileinput.input("poly.ert", inplace=True) as fin: - for line in fin: - if "NUM_REALIZATIONS" in line: - # Decrease the number of realizations to speed up the test, - # if there is flakyness, this can be increased. - print("NUM_REALIZATIONS 20", end="\n") - else: - print(line, end="") - config = ErtConfig.from_file("poly.ert") - poly_case = EnKFMain(config) - args_mock = Mock() - args_mock.config = "poly.ert" - - with StorageService.init_service( - project=os.path.abspath(config.ens_path), - ), open_storage(config.ens_path, mode="w") as storage: - gui = _setup_main_window(poly_case, args_mock, GUILogHandler(), storage) - gui.notifier.set_current_ensemble( - storage.create_experiment( - parameters=config.ensemble_config.parameter_configuration, - observations=config.observations, - ).create_ensemble( - name="default", - ensemble_size=config.model_config.num_realizations, - ) - ) - yield gui - gui.close() + ) + + +@contextmanager +def _open_main_window( + path, +) -> Generator[Tuple[ErtMainWindow, Storage, ErtConfig], None, None]: + config = ErtConfig.from_file(path / "poly.ert") + poly_case = EnKFMain(config) + + args_mock = Mock() + args_mock.config = "poly.ert" + with open_storage(config.ens_path, mode="w") as storage: + gui = _setup_main_window(poly_case, args_mock, GUILogHandler(), storage) + yield gui, storage, config + gui.close() @pytest.fixture -def opened_main_window_clean(source_root, tmpdir): - with pytest.MonkeyPatch.context() as mp: - shutil.copytree( - os.path.join(source_root, "test-data", "poly_example"), - tmpdir / "test_data", - ) - mp.chdir(tmpdir / "test_data") +def opened_main_window_clean(source_root, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _new_poly_example(source_root, tmp_path) + with _open_main_window(tmp_path) as (gui, _, config), StorageService.init_service( + project=os.path.abspath(config.ens_path), + ): + yield gui - with fileinput.input("poly.ert", inplace=True) as fin: - for line in fin: - if "NUM_REALIZATIONS" in line: - # Decrease the number of realizations to speed up the test, - # if there is flakyness, this can be increased. - print("NUM_REALIZATIONS 20", end="\n") - else: - print(line, end="") - poly_case = EnKFMain(ErtConfig.from_file("poly.ert")) - args_mock = Mock() - args_mock.config = "poly.ert" +@pytest.fixture(scope="module") +def _esmda_run(run_experiment, source_root, tmp_path_factory): + path = tmp_path_factory.mktemp("test-data") + _new_poly_example(source_root, path) + with pytest.MonkeyPatch.context() as mp, _open_main_window(path) as ( + gui, + storage, + config, + ): + mp.chdir(path) + _add_default_ensemble(storage, gui, config) + run_experiment(MultipleDataAssimilation, gui) - with StorageService.init_service( - project=os.path.abspath(poly_case.ert_config.ens_path), - ), open_storage(poly_case.ert_config.ens_path, mode="w") as storage: - gui = _setup_main_window(poly_case, args_mock, GUILogHandler(), storage) - yield gui + return path @pytest.fixture(scope="module") -def esmda_has_run(run_experiment): - # Runs a default ES-MDA run - run_experiment(MultipleDataAssimilation) +def _ensemble_experiment_run(run_experiment, source_root, tmp_path_factory): + path = tmp_path_factory.mktemp("test-data") + _new_poly_example(source_root, path) + with pytest.MonkeyPatch.context() as mp, _open_main_window(path) as ( + gui, + storage, + config, + ): + mp.chdir(path) + with open("poly_eval.py", "w", encoding="utf-8") as f: + f.write( + dedent( + """\ + #!/usr/bin/env python3 + import numpy as np + import sys + import json + + def _load_coeffs(filename): + with open(filename, encoding="utf-8") as f: + return json.load(f)["COEFFS"] + + def _evaluate(coeffs, x): + return coeffs["a"] * x**2 + coeffs["b"] * x + coeffs["c"] + + if __name__ == "__main__": + if np.random.random(1) > 0.5: + sys.exit(1) + coeffs = _load_coeffs("parameters.json") + output = [_evaluate(coeffs, x) for x in range(10)] + with open("poly.out", "w", encoding="utf-8") as f: + f.write("\\n".join(map(str, output))) + """ + ) + ) + os.chmod( + "poly_eval.py", + os.stat("poly_eval.py").st_mode + | stat.S_IXUSR + | stat.S_IXGRP + | stat.S_IXOTH, + ) + _add_default_ensemble(storage, gui, config) + run_experiment(EnsembleExperiment, gui) + + return path + + +@pytest.fixture +def esmda_has_run(_esmda_run, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + shutil.copytree(_esmda_run, tmp_path, dirs_exist_ok=True) + with _open_main_window(tmp_path) as ( + gui, + _, + config, + ), StorageService.init_service( + project=os.path.abspath(config.ens_path), + ): + yield gui + + +@pytest.fixture +def ensemble_experiment_has_run(_ensemble_experiment_run, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + shutil.copytree(_ensemble_experiment_run, tmp_path, dirs_exist_ok=True) + with _open_main_window(tmp_path) as ( + gui, + _, + config, + ), StorageService.init_service( + project=os.path.abspath(config.ens_path), + ): + yield gui @pytest.fixture(name="run_experiment", scope="module") -def run_experiment_fixture(request, opened_main_window): - def func(experiment_mode): +def run_experiment_fixture(request): + def func(experiment_mode, gui): qtbot = QtBot(request) - gui = opened_main_window with contextlib.suppress(FileNotFoundError): shutil.rmtree("poly_out") # Select correct experiment in the simulation panel @@ -148,6 +237,11 @@ def func(experiment_mode): simulation_mode_combo = simulation_panel.findChild(QComboBox) assert isinstance(simulation_mode_combo, QComboBox) simulation_mode_combo.setCurrentText(experiment_mode.name()) + simulation_settings = simulation_panel._simulation_widgets[ + simulation_panel.getCurrentSimulationModel() + ] + if hasattr(simulation_settings, "_ensemble_name_field"): + simulation_settings._ensemble_name_field.setText("iter-0") # Click start simulation and agree to the message start_simulation = simulation_panel.findChild(QWidget, name="start_simulation") @@ -158,10 +252,14 @@ def handle_dialog(): ) QTimer.singleShot( - 500, lambda: handle_run_path_dialog(gui, qtbot, delete_run_path=False) + 500, + lambda: handle_run_path_dialog(gui, qtbot, delete_run_path=False), ) - if not experiment_mode.name() in ("Ensemble experiment", "Evaluate ensemble"): + if not experiment_mode.name() in ( + "Ensemble experiment", + "Evaluate ensemble", + ): QTimer.singleShot(500, handle_dialog) qtbot.mouseClick(start_simulation, Qt.LeftButton) @@ -190,50 +288,6 @@ def handle_dialog(): return func -@pytest.fixture(scope="module") -def ensemble_experiment_has_run(opened_main_window, run_experiment, request): - gui = opened_main_window - - simulation_panel = get_child(gui, SimulationPanel) - simulation_mode_combo = get_child(simulation_panel, QComboBox) - simulation_settings = get_child(simulation_panel, EnsembleExperimentPanel) - simulation_mode_combo.setCurrentText(EnsembleExperiment.name()) - - simulation_settings._ensemble_name_field.setText("iter-0") - - with open("poly_eval.py", "w", encoding="utf-8") as f: - f.write( - dedent( - """\ - #!/usr/bin/env python - import numpy as np - import sys - import json - - def _load_coeffs(filename): - with open(filename, encoding="utf-8") as f: - return json.load(f)["COEFFS"] - - def _evaluate(coeffs, x): - return coeffs["a"] * x**2 + coeffs["b"] * x + coeffs["c"] - - if __name__ == "__main__": - if np.random.random(1) > 0.5: - sys.exit(1) - coeffs = _load_coeffs("parameters.json") - output = [_evaluate(coeffs, x) for x in range(10)] - with open("poly.out", "w", encoding="utf-8") as f: - f.write("\\n".join(map(str, output))) - """ - ) - ) - os.chmod( - "poly_eval.py", - os.stat("poly_eval.py").st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, - ) - run_experiment(EnsembleExperiment) - - @pytest.fixture() def full_snapshot() -> Snapshot: real = RealizationSnapshot( @@ -301,8 +355,8 @@ def large_snapshot() -> Snapshot: status=FORWARD_MODEL_STATE_START, stdout=f"job_{i}.stdout", stderr=f"job_{i}.stderr", - start_time=dt(1999, 1, 1).isoformat(), - end_time=dt(2019, 1, 1).isoformat(), + start_time=dt(1999, 1, 1), + end_time=dt(2019, 1, 1), ) real_ids = [str(i) for i in range(0, 150)] return builder.build(real_ids, REALIZATION_STATE_UNKNOWN) @@ -321,8 +375,8 @@ def small_snapshot() -> Snapshot: status=FORWARD_MODEL_STATE_START, stdout=f"job_{i}.stdout", stderr=f"job_{i}.stderr", - start_time=dt(1999, 1, 1).isoformat(), - end_time=dt(2019, 1, 1).isoformat(), + start_time=dt(1999, 1, 1), + end_time=dt(2019, 1, 1), ) real_ids = [str(i) for i in range(0, 5)] return builder.build(real_ids, REALIZATION_STATE_UNKNOWN) diff --git a/tests/unit_tests/gui/test_full_manual_update_workflow.py b/tests/unit_tests/gui/test_full_manual_update_workflow.py index 55a16273562..2c5ba511966 100644 --- a/tests/unit_tests/gui/test_full_manual_update_workflow.py +++ b/tests/unit_tests/gui/test_full_manual_update_workflow.py @@ -23,16 +23,20 @@ from .conftest import get_child, wait_for_child, with_manage_tool -def test_that_the_manual_analysis_tool_works( - ensemble_experiment_has_run, opened_main_window, qtbot, run_experiment -): +def test_that_the_manual_analysis_tool_works(ensemble_experiment_has_run, qtbot): """This runs a full manual update workflow, first running ensemble experiment where some of the realizations fail, then doing an update before running an ensemble experiment again to calculate the forecast of the update. """ - gui = opened_main_window + gui = ensemble_experiment_has_run analysis_tool = gui.tools["Run analysis"] + # Select correct experiment in the simulation panel + simulation_panel = get_child(gui, SimulationPanel) + simulation_settings = get_child(simulation_panel, EvaluateEnsemblePanel) + simulation_mode_combo = get_child(simulation_panel, QComboBox) + simulation_mode_combo.setCurrentText(EvaluateEnsemble.name()) + # Open the "Run analysis" tool in the main window after ensemble experiment has run def handle_analysis_dialog(): dialog = analysis_tool._dialog @@ -43,12 +47,9 @@ def handle_analysis_dialog(): # Source case is "iter-0" ensemble_selector = run_panel.source_ensemble_selector - current_select = 0 - ensemble_selector.setCurrentIndex(current_select) - while ensemble_selector.currentText() != "iter-0": - current_select += 1 - simulation_settings._ensemble_selector.setCurrentIndex(current_select) - assert ensemble_selector.currentText().startswith("iter-0") + idx = ensemble_selector.findData("iter-0", Qt.MatchStartsWith) + assert idx != -1 + ensemble_selector.setCurrentIndex(idx) # Click on "Run" and click ok on the message box def handle_dialog(): @@ -79,29 +80,20 @@ def handle_manage_dialog(dialog, cases_panel): tree_view = get_child(storage_widget, QTreeView) tree_view.expandAll() - assert tree_view.model().rowCount() == 2 - assert "iter-1" in tree_view.model().index( - 1, 0, tree_view.model().index(1, 0) - ).data(0) + model = tree_view.model() + assert model is not None and model.rowCount() == 2 + assert "iter-1" in model.index(1, 0, model.index(1, 0)).data(0) dialog.close() with_manage_tool(gui, qtbot, handle_manage_dialog) - # Select correct experiment in the simulation panel - simulation_panel = get_child(gui, SimulationPanel) - simulation_mode_combo = get_child(simulation_panel, QComboBox) - simulation_settings = get_child(simulation_panel, EvaluateEnsemblePanel) - simulation_mode_combo.setCurrentText(EvaluateEnsemble.name()) - with contextlib.suppress(FileNotFoundError): shutil.rmtree("poly_out") - current_select = 0 - simulation_settings._ensemble_selector.setCurrentIndex(current_select) - while simulation_settings._ensemble_selector.currentText() != "iter-1": - current_select += 1 - simulation_settings._ensemble_selector.setCurrentIndex(current_select) + idx = simulation_settings._ensemble_selector.findData("iter-1", Qt.MatchStartsWith) + assert idx != -1 + simulation_settings._ensemble_selector.setCurrentIndex(idx) storage = gui.notifier.storage ensemble_prior = storage.get_ensemble_by_name("iter-0") diff --git a/tests/unit_tests/gui/test_main_window.py b/tests/unit_tests/gui/test_main_window.py index f48358a9b1f..46e6444e697 100644 --- a/tests/unit_tests/gui/test_main_window.py +++ b/tests/unit_tests/gui/test_main_window.py @@ -384,11 +384,11 @@ def test_that_ert_changes_to_config_directory(qtbot): assert gui.windowTitle() == "ERT - snake_oil_surface.ert" -@pytest.mark.usefixtures("esmda_has_run", "use_tmpdir", "using_scheduler") +@pytest.mark.usefixtures("using_scheduler") def test_that_the_plot_window_contains_the_expected_elements( - opened_main_window: ErtMainWindow, qtbot + esmda_has_run: ErtMainWindow, qtbot ): - gui = opened_main_window + gui = esmda_has_run expected_ensembles = [ "default", "default_0", @@ -447,8 +447,10 @@ def test_that_the_plot_window_contains_the_expected_elements( # Cycle through showing all the tabs and plot each data key - for i in range(data_keys.model().rowCount()): - index = data_keys.model().index(i, 0) + model = data_keys.model() + assert model is not None + for i in range(model.rowCount()): + index = model.index(i, 0) qtbot.mouseClick( data_types.data_type_keys_widget, Qt.LeftButton, @@ -461,13 +463,11 @@ def test_that_the_plot_window_contains_the_expected_elements( plot_window.close() -@pytest.mark.usefixtures("use_tmpdir") def test_that_the_manage_experiments_tool_can_be_used( esmda_has_run, - opened_main_window, qtbot, ): - gui = opened_main_window + gui = esmda_has_run # Click on "Manage Experiments" in the main window def handle_dialog(dialog, experiments_panel): @@ -500,13 +500,6 @@ def handle_add_dialog(): experiments_panel.setCurrentIndex(1) current_tab = experiments_panel.currentWidget() assert current_tab.objectName() == "initialize_from_scratch_panel" - combo_box = get_child(current_tab, EnsembleSelector) - - # Select "new_case" - current_index = 0 - while combo_box.currentText().startswith("new_case"): - current_index += 1 - combo_box.setCurrentIndex(current_index) # click on "initialize" initialize_button = get_child( @@ -521,7 +514,6 @@ def handle_add_dialog(): with_manage_tool(gui, qtbot, handle_dialog) -@pytest.mark.usefixtures("use_tmpdir") def test_that_inversion_type_can_be_set_from_gui(qtbot, opened_main_window): gui = opened_main_window @@ -552,11 +544,8 @@ def handle_analysis_module_panel(): @pytest.mark.filterwarnings("ignore:.*Use load_responses.*:DeprecationWarning") -@pytest.mark.usefixtures("use_tmpdir") -def test_that_csv_export_plugin_generates_a_file( - qtbot, esmda_has_run, opened_main_window -): - gui = opened_main_window +def test_that_csv_export_plugin_generates_a_file(qtbot, esmda_has_run): + gui = esmda_has_run # Find EXPORT_CSV in the plugin menu plugin_tool = gui.tools["Plugins"] @@ -649,13 +638,11 @@ def handle_add_dialog(): @pytest.mark.usefixtures("use_tmpdir") -def test_that_load_results_manually_can_be_run_after_esmda( - esmda_has_run, opened_main_window, qtbot -): - load_results_manually(qtbot, opened_main_window) +def test_that_load_results_manually_can_be_run_after_esmda(esmda_has_run, qtbot): + load_results_manually(qtbot, esmda_has_run) -@pytest.mark.usefixtures("use_tmpdir", "using_scheduler") +@pytest.mark.usefixtures("using_scheduler") def test_that_a_failing_job_shows_error_message_with_context( opened_main_window_clean, qtbot ):