diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 4d05ab54..997b0857 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -9,10 +9,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - run: python -m pip install poetry + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + - name: "Set up Python" + run: uv python install - run: | - python -m poetry install --only docs - poetry run mkdocs gh-deploy --force + uv sync --only-group docs + uv run mkdocs gh-deploy --force diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ffbaaa09..ba676007 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,10 +15,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - run: python -m pip install poetry - run: | # Download the binary curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.5.4/argo-linux-amd64.gz @@ -39,9 +35,7 @@ jobs: run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: "Set up Python" - uses: actions/setup-python@v5 - with: - python-version-file: ".python-version" + run: uv python install - name: Install the project run: uv sync --all-extras --dev diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3588538d..a3fb532f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,13 +18,12 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - run: python -m pip install poetry + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + - name: "Set up Python" + run: uv python install - run: | uv sync --only-group release - - name: Figure version continue-on-error: true env: diff --git a/tests/extensions/catalog/test_catalog_extension.py b/tests/extensions/catalog/test_catalog_extension.py index ae1ededc..685b6dba 100644 --- a/tests/extensions/catalog/test_catalog_extension.py +++ b/tests/extensions/catalog/test_catalog_extension.py @@ -1,4 +1,4 @@ -from extensions.catalog import is_catalog_out_of_sync +from runnable.catalog import is_catalog_out_of_sync def test_is_catalog_out_of_sync_returns_true_for_empty_synced_catalogs(): diff --git a/tests/extensions/catalog/test_file_system.py b/tests/extensions/catalog/test_file_system.py index 0b43ba68..f6b8036a 100644 --- a/tests/extensions/catalog/test_file_system.py +++ b/tests/extensions/catalog/test_file_system.py @@ -3,8 +3,8 @@ import pytest -import extensions.catalog.file_system as implementation -from extensions.catalog.file_system.implementation import FileSystemCatalog +from extensions.catalog import file_system as module +from extensions.catalog.file_system import FileSystemCatalog from runnable import defaults @@ -38,7 +38,7 @@ def mock_does_dir_exist(dir_name): return True return False - monkeypatch.setattr(implementation.utils, "does_dir_exist", mock_does_dir_exist) + monkeypatch.setattr(module.utils, "does_dir_exist", mock_does_dir_exist) catalog_handler = FileSystemCatalog(catalog_location="this_location") with pytest.raises(Exception, match="Expected Catalog to be present at"): @@ -62,13 +62,13 @@ def test_file_system_catalog_get_copies_files_from_catalog_to_compute_folder_wit with tempfile.TemporaryDirectory() as catalog_location: with tempfile.TemporaryDirectory(dir=".") as compute_folder: - catalog_location_path = implementation.Path(catalog_location) + catalog_location_path = module.Path(catalog_location) run_id = "testing" - implementation.Path(catalog_location_path / run_id / compute_folder).mkdir( + module.Path(catalog_location_path / run_id / compute_folder).mkdir( parents=True ) with open( - implementation.Path(catalog_location) + module.Path(catalog_location) / run_id / compute_folder / "catalog_file", @@ -101,13 +101,13 @@ def test_file_system_catalog_get_copies_files_from_catalog_to_compute_folder_wit with tempfile.TemporaryDirectory() as catalog_location: with tempfile.TemporaryDirectory(dir=".") as compute_folder: - catalog_location_path = implementation.Path(catalog_location) + catalog_location_path = module.Path(catalog_location) run_id = "testing" - implementation.Path(catalog_location_path / run_id / compute_folder).mkdir( + module.Path(catalog_location_path / run_id / compute_folder).mkdir( parents=True ) with open( - implementation.Path(catalog_location) + module.Path(catalog_location) / run_id / compute_folder / "catalog_file", @@ -116,10 +116,7 @@ def test_file_system_catalog_get_copies_files_from_catalog_to_compute_folder_wit fw.write("hello") with open( - implementation.Path(catalog_location) - / run_id - / compute_folder - / "not_catalog", + module.Path(catalog_location) / run_id / compute_folder / "not_catalog", "w", ) as fw: fw.write("hello") @@ -136,7 +133,7 @@ def test_file_system_catalog_put_copies_files_from_compute_folder_to_catalog_if_ mocker, monkeypatch ): monkeypatch.setattr( - implementation, "is_catalog_out_of_sync", mocker.MagicMock(return_value=True) + module, "is_catalog_out_of_sync", mocker.MagicMock(return_value=True) ) mock_run_store = mocker.MagicMock() mock_context = mocker.MagicMock() @@ -150,11 +147,11 @@ def test_file_system_catalog_put_copies_files_from_compute_folder_to_catalog_if_ with tempfile.TemporaryDirectory() as catalog_location: with tempfile.TemporaryDirectory(dir=".") as compute_folder: - catalog_location_path = implementation.Path(catalog_location) + catalog_location_path = module.Path(catalog_location) run_id = "testing" - implementation.Path(catalog_location_path / run_id).mkdir(parents=True) + module.Path(catalog_location_path / run_id).mkdir(parents=True) - with open(implementation.Path(compute_folder) / "catalog_file", "w") as fw: + with open(module.Path(compute_folder) / "catalog_file", "w") as fw: fw.write("hello") catalog_handler = FileSystemCatalog(catalog_location=catalog_location) @@ -169,7 +166,7 @@ def test_file_system_catalog_put_copies_files_from_compute_folder_to_catalog_if_ mocker, monkeypatch ): monkeypatch.setattr( - implementation, "is_catalog_out_of_sync", mocker.MagicMock(return_value=True) + module, "is_catalog_out_of_sync", mocker.MagicMock(return_value=True) ) mock_run_store = mocker.MagicMock() mock_context = mocker.MagicMock() @@ -182,15 +179,13 @@ def test_file_system_catalog_put_copies_files_from_compute_folder_to_catalog_if_ ) with tempfile.TemporaryDirectory() as catalog_location: with tempfile.TemporaryDirectory(dir=".") as compute_folder: - catalog_location_path = implementation.Path(catalog_location) + catalog_location_path = module.Path(catalog_location) run_id = "testing" - implementation.Path(catalog_location_path / run_id).mkdir(parents=True) - with open(implementation.Path(compute_folder) / "catalog_file", "w") as fw: + module.Path(catalog_location_path / run_id).mkdir(parents=True) + with open(module.Path(compute_folder) / "catalog_file", "w") as fw: fw.write("hello") - with open( - implementation.Path(compute_folder) / "not_catalog_file", "w" - ) as fw: + with open(module.Path(compute_folder) / "not_catalog_file", "w") as fw: fw.write("hello") catalog_handler = FileSystemCatalog(catalog_location=catalog_location) @@ -206,7 +201,7 @@ def test_file_system_catalog_put_copies_files_from_compute_folder_to_catalog_if_ mocker, monkeypatch ): monkeypatch.setattr( - implementation, "is_catalog_out_of_sync", mocker.MagicMock(return_value=False) + module, "is_catalog_out_of_sync", mocker.MagicMock(return_value=False) ) mock_run_store = mocker.MagicMock() mock_context = mocker.MagicMock() @@ -220,15 +215,13 @@ def test_file_system_catalog_put_copies_files_from_compute_folder_to_catalog_if_ with tempfile.TemporaryDirectory() as catalog_location: with tempfile.TemporaryDirectory(dir=".") as compute_folder: - catalog_location_path = implementation.Path(catalog_location) + catalog_location_path = module.Path(catalog_location) run_id = "testing" - implementation.Path(catalog_location_path / run_id).mkdir(parents=True) - with open(implementation.Path(compute_folder) / "catalog_file", "w") as fw: + module.Path(catalog_location_path / run_id).mkdir(parents=True) + with open(module.Path(compute_folder) / "catalog_file", "w") as fw: fw.write("hello") - with open( - implementation.Path(compute_folder) / "not_catalog_file", "w" - ) as fw: + with open(module.Path(compute_folder) / "not_catalog_file", "w") as fw: fw.write("hello") catalog_handler = FileSystemCatalog(catalog_location=catalog_location) @@ -242,24 +235,24 @@ def test_file_system_catalog_put_copies_files_from_compute_folder_to_catalog_if_ def test_file_system_catalog_put_uses_compute_folder_by_default(monkeypatch, mocker): mock_safe_make_dir = mocker.MagicMock() - monkeypatch.setattr(implementation.utils, "safe_make_dir", mock_safe_make_dir) + monkeypatch.setattr(module.utils, "safe_make_dir", mock_safe_make_dir) mock_does_dir_exist = mocker.MagicMock(side_effect=Exception()) - monkeypatch.setattr(implementation.utils, "does_dir_exist", mock_does_dir_exist) + monkeypatch.setattr(module.utils, "does_dir_exist", mock_does_dir_exist) catalog_handler = FileSystemCatalog(catalog_location="this_location") with pytest.raises(Exception): catalog_handler.put("testing", run_id="dummy_run_id") - mock_does_dir_exist.assert_called_once_with(implementation.Path(".")) + mock_does_dir_exist.assert_called_once_with(module.Path(".")) def test_file_system_catalog_put_uses_compute_folder_provided(monkeypatch, mocker): mock_safe_make_dir = mocker.MagicMock() - monkeypatch.setattr(implementation.utils, "safe_make_dir", mock_safe_make_dir) + monkeypatch.setattr(module.utils, "safe_make_dir", mock_safe_make_dir) mock_does_dir_exist = mocker.MagicMock(side_effect=Exception()) - monkeypatch.setattr(implementation.utils, "does_dir_exist", mock_does_dir_exist) + monkeypatch.setattr(module.utils, "does_dir_exist", mock_does_dir_exist) catalog_handler = FileSystemCatalog(catalog_location="this_location") with pytest.raises(Exception): @@ -267,17 +260,17 @@ def test_file_system_catalog_put_uses_compute_folder_provided(monkeypatch, mocke "testing", run_id="dummy_run_id", compute_data_folder="not_data" ) - mock_does_dir_exist.assert_called_once_with(implementation.Path("not_data")) + mock_does_dir_exist.assert_called_once_with(module.Path("not_data")) def test_file_system_catalog_put_raises_exception_if_compute_data_folder_does_not_exist( monkeypatch, mocker ): mock_safe_make_dir = mocker.MagicMock() - monkeypatch.setattr(implementation.utils, "safe_make_dir", mock_safe_make_dir) + monkeypatch.setattr(module.utils, "safe_make_dir", mock_safe_make_dir) mock_does_dir_exist = mocker.MagicMock(return_value=False) - monkeypatch.setattr(implementation.utils, "does_dir_exist", mock_does_dir_exist) + monkeypatch.setattr(module.utils, "does_dir_exist", mock_does_dir_exist) catalog_handler = FileSystemCatalog(catalog_location="this_location") with pytest.raises(Exception): @@ -290,10 +283,10 @@ def test_file_system_catalog_put_creates_catalog_location_using_run_id( monkeypatch, mocker ): mock_safe_make_dir = mocker.MagicMock() - monkeypatch.setattr(implementation.utils, "safe_make_dir", mock_safe_make_dir) + monkeypatch.setattr(module.utils, "safe_make_dir", mock_safe_make_dir) mock_does_dir_exist = mocker.MagicMock(side_effect=Exception()) - monkeypatch.setattr(implementation.utils, "does_dir_exist", mock_does_dir_exist) + monkeypatch.setattr(module.utils, "does_dir_exist", mock_does_dir_exist) catalog_handler = FileSystemCatalog(catalog_location="this_location") @@ -301,7 +294,7 @@ def test_file_system_catalog_put_creates_catalog_location_using_run_id( catalog_handler.put("testing", run_id="dummy_run_id") mock_safe_make_dir.assert_called_once_with( - implementation.Path("this_location") / "dummy_run_id" + module.Path("this_location") / "dummy_run_id" ) @@ -309,10 +302,10 @@ def test_file_system_sync_between_runs_raises_exception_if_previous_catalog_does monkeypatch, mocker ): mock_safe_make_dir = mocker.MagicMock() - monkeypatch.setattr(implementation.utils, "safe_make_dir", mock_safe_make_dir) + monkeypatch.setattr(module.utils, "safe_make_dir", mock_safe_make_dir) mock_does_dir_exist = mocker.MagicMock(return_value=False) - monkeypatch.setattr(implementation.utils, "does_dir_exist", mock_does_dir_exist) + monkeypatch.setattr(module.utils, "does_dir_exist", mock_does_dir_exist) catalog_handler = FileSystemCatalog(catalog_location="this_location") with pytest.raises(Exception): diff --git a/tests/extensions/catalog/test_k8s_pvc.py b/tests/extensions/catalog/test_k8s_pvc.py deleted file mode 100644 index 81a3dfa3..00000000 --- a/tests/extensions/catalog/test_k8s_pvc.py +++ /dev/null @@ -1,11 +0,0 @@ -from runnable.extensions.catalog.k8s_pvc.implementation import K8sPVCatalog - - -def test_get_catalog_location_returns_location_relative_to_mount_path(): - test_catalog = K8sPVCatalog( - catalog_location="test_location", - mount_path="/here", - persistent_volume_name="test", - ) - - assert test_catalog.get_catalog_location() == "/here/test_location" diff --git a/tests/extensions/catalog/test_k8s_pvc_integration.py b/tests/extensions/catalog/test_k8s_pvc_integration.py deleted file mode 100644 index a174340c..00000000 --- a/tests/extensions/catalog/test_k8s_pvc_integration.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from runnable.extensions.catalog.k8s_pvc import integration - - -def test_k8s_pvc_errors_for_local(): - test_integration = integration.LocalCompute( - executor="executor", integration_service="catalog" - ) - - with pytest.raises(Exception, match="We can't use the local compute"): - test_integration.validate() - - -def test_k8s_pvc_errors_for_local_container(): - test_integration = integration.LocalContainerCompute( - executor="executor", integration_service="catalog" - ) - - with pytest.raises(Exception, match="We can't use the local-container compute"): - test_integration.validate() diff --git a/tests/extensions/executor/test_argo_executor.py b/tests/extensions/executor/test_argo_executor.py index c8b62a4e..b86103fc 100644 --- a/tests/extensions/executor/test_argo_executor.py +++ b/tests/extensions/executor/test_argo_executor.py @@ -1,6 +1,6 @@ import pytest -from runnable.extensions.executor.argo import implementation +from extensions.executor import argo as implementation def test_secret_env_var_has_value_from_field(): diff --git a/tests/extensions/executor/test_generic_executor.py b/tests/extensions/executor/test_generic_executor.py index 42a4a442..0286a000 100644 --- a/tests/extensions/executor/test_generic_executor.py +++ b/tests/extensions/executor/test_generic_executor.py @@ -1,23 +1,32 @@ -import logging - import pytest -import runnable.extensions.executor as executor from runnable import defaults, exceptions -from runnable.extensions import executor -from runnable.extensions.executor import GenericExecutor @pytest.fixture(autouse=True) -def instantiable_base_class(monkeypatch, mocker): +def instantiable_base_class(monkeypatch): monkeypatch.setattr(GenericExecutor, "__abstractmethods__", set()) yield +@pytest.fixture +def mock_base_node(monkeypatch, mocker): + from runnable.nodes import BaseNode + + monkeypatch.setattr(BaseNode, "__abstractmethods__", set()) + + class MockNode(BaseNode): + node_type: str = "test" + name: str = "name" + internal_name: str = "internal_name" + + yield MockNode + + @pytest.fixture def mock_run_context(mocker, monkeypatch): mock_run_context = mocker.Mock() - monkeypatch.setattr(executor.context, "run_context", mock_run_context) + monkeypatch.setattr(module.context, "run_context", mock_run_context) return mock_run_context @@ -26,7 +35,7 @@ def test_get_parameters_gets_parameters_from_user_parameters( ): mock_run_context.parameters_file = "" monkeypatch.setattr( - executor.parameters, + module.parameters, "get_user_set_parameters", mocker.MagicMock(return_value={"executor": "test"}), ) @@ -41,9 +50,9 @@ def test_get_parameters_user_parameters_overwrites_parameters_from_parameters_fi mock_run_context.parameters_file = "parameters_file" mock_load_yaml = mocker.MagicMock(return_value={"executor": "this"}) - monkeypatch.setattr(executor.utils, "load_yaml", mock_load_yaml) + monkeypatch.setattr(module.utils, "load_yaml", mock_load_yaml) monkeypatch.setattr( - executor.parameters, + module.parameters, "get_user_set_parameters", mocker.MagicMock(return_value={"executor": "that"}), ) @@ -161,8 +170,8 @@ def test_base_executor_prepare_for_graph_execution_calls( mock_set_up_run_log = mocker.MagicMock() monkeypatch.setattr(GenericExecutor, "_set_up_run_log", mock_set_up_run_log) - monkeypatch.setattr(executor, "integration", mock_integration) - monkeypatch.setattr(executor.BaseExecutor, "_set_up_run_log", mocker.MagicMock()) + monkeypatch.setattr(module, "integration", mock_integration) + monkeypatch.setattr(module.BaseExecutor, "_set_up_run_log", mocker.MagicMock()) base_executor = GenericExecutor() @@ -183,7 +192,7 @@ def test_base_execution_prepare_for_node_calls(mocker, monkeypatch, mock_run_con mock_set_up_run_log = mocker.MagicMock() monkeypatch.setattr(GenericExecutor, "_set_up_run_log", mock_set_up_run_log) - monkeypatch.setattr(executor, "integration", mock_integration) + monkeypatch.setattr(module, "integration", mock_integration) base_executor = GenericExecutor() @@ -202,9 +211,9 @@ def test_base_executor__sync_catalog_raises_exception_if_stage_not_in_get_or_put def test_sync_catalog_does_nothing_for_terminal_node( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() + mock_node = mock_base_node() mock_node._get_catalog_settings = mocker.MagicMock( side_effect=exceptions.TerminalNodeError ) @@ -216,9 +225,9 @@ def test_sync_catalog_does_nothing_for_terminal_node( def test_sync_catalog_does_nothing_for_no_catalog_settings( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() + mock_node = mock_base_node() mock_node._get_catalog_settings = mocker.MagicMock(return_value={}) test_executor = GenericExecutor() @@ -228,9 +237,9 @@ def test_sync_catalog_does_nothing_for_no_catalog_settings( def test_sync_catalog_does_nothing_for_catalog_settings_stage_not_in( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() + mock_node = mock_base_node() mock_node._get_catalog_settings = mocker.MagicMock( return_value={"get": "something"} ) @@ -242,11 +251,10 @@ def test_sync_catalog_does_nothing_for_catalog_settings_stage_not_in( def test_sync_catalog_returns_nothing_if_no_syncing_for_node( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - - mock_node._get_catalog_settings.return_value = None + mock_node = mock_base_node() + setattr(mock_node, "_get_catalog_settings", lambda: None) test_executor = GenericExecutor() test_executor._context_node = mock_node @@ -255,10 +263,10 @@ def test_sync_catalog_returns_nothing_if_no_syncing_for_node( def test_sync_catalog_returns_empty_list_if_asked_nothing_in_stage( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_catalog_settings.return_value = {"get": [], "put": []} + mock_node = mock_base_node() + setattr(mock_node, "_get_catalog_settings", lambda: {"get": [], "put": []}) mock_get_effective_compute_folder = mocker.MagicMock(return_value="compute_folder") monkeypatch.setattr( @@ -275,11 +283,10 @@ def test_sync_catalog_returns_empty_list_if_asked_nothing_in_stage( def test_sync_catalog_calls_get_from_catalog_handler( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_catalog_settings.return_value = {"get": ["me"], "put": []} - mock_step_log = mocker.MagicMock() + mock_node = mock_base_node() + setattr(mock_node, "_get_catalog_settings", lambda: {"get": ["me"], "put": []}) mock_get_effective_compute_folder = mocker.MagicMock(return_value="compute_folder") monkeypatch.setattr( @@ -305,11 +312,12 @@ def test_sync_catalog_calls_get_from_catalog_handler( def test_sync_catalog_calls_get_from_catalog_handler_as_per_input( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_catalog_settings.return_value = {"get": ["me", "you"], "put": []} - mock_step_log = mocker.MagicMock() + mock_node = mock_base_node() + setattr( + mock_node, "_get_catalog_settings", lambda: {"get": ["me", "you"], "put": []} + ) mock_get_effective_compute_folder = mocker.MagicMock(return_value="compute_folder") monkeypatch.setattr( @@ -333,11 +341,10 @@ def test_sync_catalog_calls_get_from_catalog_handler_as_per_input( def test_sync_catalog_calls_put_from_catalog_handler( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_catalog_settings.return_value = {"get": [], "put": ["me"]} - mock_step_log = mocker.MagicMock() + mock_node = mock_base_node() + setattr(mock_node, "_get_catalog_settings", lambda: {"get": [], "put": ["me"]}) mock_get_effective_compute_folder = mocker.MagicMock(return_value="compute_folder") monkeypatch.setattr( @@ -366,11 +373,12 @@ def test_sync_catalog_calls_put_from_catalog_handler( def test_sync_catalog_calls_put_from_catalog_handler_as_per_input( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_catalog_settings.return_value = {"get": [], "put": ["me", "you"]} - mock_step_log = mocker.MagicMock() + mock_node = mock_base_node() + setattr( + mock_node, "_get_catalog_settings", lambda: {"get": [], "put": ["me", "you"]} + ) mock_get_effective_compute_folder = mocker.MagicMock(return_value="compute_folder") monkeypatch.setattr( @@ -394,11 +402,10 @@ def test_sync_catalog_calls_put_from_catalog_handler_as_per_input( def test_sync_catalog_calls_put_sends_synced_catalogs_to_catalog_handler( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_catalog_settings.return_value = {"get": [], "put": ["me"]} - mock_step_log = mocker.MagicMock() + mock_node = mock_base_node() + setattr(mock_node, "_get_catalog_settings", lambda: {"get": [], "put": ["me"]}) mock_get_effective_compute_folder = mocker.MagicMock(return_value="compute_folder") monkeypatch.setattr( @@ -426,11 +433,13 @@ def test_sync_catalog_calls_put_sends_synced_catalogs_to_catalog_handler( ) -def test_get_effective_compute_data_folder_returns_default(mocker, mock_run_context): +def test_get_effective_compute_data_folder_returns_default( + mocker, mock_run_context, mock_base_node +): mock_run_context.catalog_handler.compute_data_folder = "default" - mock_node = mocker.MagicMock() - mock_node._get_catalog_settings.return_value = {} + mock_node = mock_base_node() + setattr(mock_node, "_get_catalog_settings", lambda: {}) test_executor = GenericExecutor() test_executor._context_node = mock_node @@ -439,14 +448,16 @@ def test_get_effective_compute_data_folder_returns_default(mocker, mock_run_cont def test_get_effective_compute_data_folder_returns_from_node_settings( - mocker, mock_run_context + mocker, mock_run_context, mock_base_node ): mock_run_context.catalog_handler.compute_data_folder = "default" - mock_node = mocker.MagicMock() - mock_node._get_catalog_settings.return_value = { - "compute_data_folder": "not_default" - } + mock_node = mock_base_node() + setattr( + mock_node, + "_get_catalog_settings", + lambda: {"compute_data_folder": "not_default"}, + ) test_executor = GenericExecutor() test_executor._context_node = mock_node @@ -469,10 +480,10 @@ def test_step_attempt_returns_from_env(monkeypatch): def test_base_executor_resolve_executor_config_gives_global_config_if_node_does_not_override( - mocker, mock_run_context + mocker, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_executor_config.return_value = {} + mock_node = mock_base_node() + setattr(mock_node, "_get_catalog_settings", lambda: {}) mock_run_context.variables = {} @@ -484,9 +495,14 @@ def test_base_executor_resolve_executor_config_gives_global_config_if_node_does_ def test_get_status_and_next_node_name_returns_empty_for_terminal_node( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() + mock_node = mock_base_node() + + def f(): + raise exceptions.TerminalNodeError() + + setattr(mock_node, "_get_catalog_settings", f) mock_node._get_next_node = mocker.MagicMock( side_effect=exceptions.TerminalNodeError ) @@ -504,10 +520,10 @@ def test_get_status_and_next_node_name_returns_empty_for_terminal_node( def test_get_status_and_next_node_name_returns_next_node_if_success( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_next_node.return_value = "next_node" + mock_node = mock_base_node() + setattr(mock_node, "_get_next_node", lambda: "next_node") mock_step_log = mocker.MagicMock() mock_step_log.status = defaults.SUCCESS @@ -522,11 +538,11 @@ def test_get_status_and_next_node_name_returns_next_node_if_success( def test_get_status_and_next_node_name_returns_terminal_node_in_case_of_failure( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_next_node.return_value = "next_node" - mock_node._get_on_failure_node.return_value = "" + mock_node = mock_base_node() + setattr(mock_node, "_get_next_node", lambda: "next node") + setattr(mock_node, "_get_on_failure_node", lambda: "") mock_run_context.run_log_store.get_step_log.return_value.status = defaults.FAIL @@ -542,11 +558,11 @@ def test_get_status_and_next_node_name_returns_terminal_node_in_case_of_failure( def test_get_status_and_next_node_name_returns_on_failure_node_if_failed( - mocker, monkeypatch, mock_run_context + mocker, monkeypatch, mock_run_context, mock_base_node ): - mock_node = mocker.MagicMock() - mock_node._get_next_node.return_value = "next_node" - mock_node._get_on_failure_node.return_value = "me_please" + mock_node = mock_base_node() + setattr(mock_node, "_get_next_node", lambda: "next_node") + setattr(mock_node, "_get_on_failure_node", lambda: "me_please") mock_run_context.run_log_store.get_step_log.return_value.status = defaults.FAIL @@ -581,3 +597,7 @@ def test_send_return_code_does_not_raise_exception_if_pipeline_execution_succeed test_executor = GenericExecutor() test_executor.send_return_code() + + +from extensions import executor as module +from extensions.executor import GenericExecutor diff --git a/tests/extensions/executor/test_local_container_integration.py b/tests/extensions/executor/test_local_container_integration.py index cc380d6e..d62d0396 100644 --- a/tests/extensions/executor/test_local_container_integration.py +++ b/tests/extensions/executor/test_local_container_integration.py @@ -1,12 +1,12 @@ from pathlib import Path -from runnable.extensions.executor.local_container import implementation +from extensions.executor import local_container as module def test_configure_for_traversal_populates_volumes(mocker, monkeypatch): mock_local_container = mocker.MagicMock() monkeypatch.setattr( - implementation, + module, "LocalContainerComputeFileSystemRunLogstore", mock_local_container, ) @@ -18,7 +18,7 @@ def test_configure_for_traversal_populates_volumes(mocker, monkeypatch): mock_fs_catalog = mocker.MagicMock() mock_fs_catalog.catalog_location = "catalog_location" - test_integration = implementation.LocalContainerComputeFileSystemCatalog( + test_integration = module.LocalContainerComputeFileSystemCatalog( mock_executor, mock_fs_catalog ) test_integration.configure_for_traversal() @@ -33,7 +33,7 @@ def test_configure_for_execution_assigns_catalog_location_within_container( ): mock_local_container = mocker.MagicMock() monkeypatch.setattr( - implementation, + module, "LocalContainerComputeFileSystemRunLogstore", mock_local_container, ) @@ -43,7 +43,7 @@ def test_configure_for_execution_assigns_catalog_location_within_container( mock_fs_catalog = mocker.MagicMock() - test_integration = implementation.LocalContainerComputeFileSystemCatalog( + test_integration = module.LocalContainerComputeFileSystemCatalog( mock_executor, mock_fs_catalog ) test_integration.configure_for_execution() diff --git a/tests/extensions/executor/test_local_executor.py b/tests/extensions/executor/test_local_executor.py index 1f0b6121..af3333a0 100644 --- a/tests/extensions/executor/test_local_executor.py +++ b/tests/extensions/executor/test_local_executor.py @@ -1,4 +1,4 @@ -from runnable.extensions.executor.local import LocalExecutor +from extensions.executor.local import LocalExecutor def test_local_executor_execute_node_just_calls___execute_node(mocker, monkeypatch): diff --git a/tests/extensions/run_log_store/test_chunked_k8s_pvc_integration.py b/tests/extensions/run_log_store/test_chunked_k8s_pvc_integration.py deleted file mode 100644 index e336aaec..00000000 --- a/tests/extensions/run_log_store/test_chunked_k8s_pvc_integration.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from runnable.extensions.run_log_store.chunked_k8s_pvc import integration - - -def test_k8s_pvc_errors_for_local(): - test_integration = integration.LocalCompute( - executor="executor", integration_service="catalog" - ) - - with pytest.raises(Exception, match="We can't use the local compute"): - test_integration.validate() - - -def test_k8s_pvc_errors_for_local_container(): - test_integration = integration.LocalContainerCompute( - executor="executor", integration_service="catalog" - ) - - with pytest.raises(Exception, match="We can't use the local-container compute"): - test_integration.validate() diff --git a/tests/extensions/run_log_store/test_file_system.py b/tests/extensions/run_log_store/test_file_system.py index 6f16b6b3..c7f7fabe 100644 --- a/tests/extensions/run_log_store/test_file_system.py +++ b/tests/extensions/run_log_store/test_file_system.py @@ -1,11 +1,8 @@ import pytest -from runnable.extensions.run_log_store.file_system.implementation import ( - FileSystemRunLogstore, -) -import runnable.extensions.run_log_store.file_system.implementation as implementation -from runnable import defaults -from runnable import exceptions +import extensions.run_log_store.file_system as module +from extensions.run_log_store.file_system import FileSystemRunLogstore +from runnable import defaults, exceptions def test_file_system_run_log_store_log_folder_name_defaults_if_not_provided(): @@ -24,12 +21,12 @@ def test_file_system_run_log_store_write_to_folder_makes_dir_if_not_present( mocker, monkeypatch ): mock_safe_make_dir = mocker.MagicMock() - monkeypatch.setattr(implementation.utils, "safe_make_dir", mock_safe_make_dir) + monkeypatch.setattr(module.utils, "safe_make_dir", mock_safe_make_dir) mock_json = mocker.MagicMock() mock_path = mocker.MagicMock() - monkeypatch.setattr(implementation, "json", mock_json) - monkeypatch.setattr(implementation, "Path", mock_path) + monkeypatch.setattr(module, "json", mock_json) + monkeypatch.setattr(module, "Path", mock_path) mock_run_log = mocker.MagicMock() mock_dict = mocker.MagicMock() @@ -46,15 +43,13 @@ def test_file_system_run_log_store_get_from_folder_raises_exception_if_folder_no mocker, monkeypatch ): mock_path = mocker.MagicMock() - monkeypatch.setattr( - implementation, "Path", mocker.MagicMock(return_value=mock_path) - ) + monkeypatch.setattr(module, "Path", mocker.MagicMock(return_value=mock_path)) mock_path.__truediv__.return_value = mock_path mock_path.exists.return_value = False - run_log_store = implementation.FileSystemRunLogstore() + run_log_store = module.FileSystemRunLogstore() with pytest.raises(FileNotFoundError): run_log_store.get_from_folder(run_id="test") @@ -64,18 +59,16 @@ def test_file_system_run_log_store_get_from_folder_returns_run_log_from_file_con mocker, monkeypatch ): mock_path = mocker.MagicMock() - monkeypatch.setattr( - implementation, "Path", mocker.MagicMock(return_value=mock_path) - ) + monkeypatch.setattr(module, "Path", mocker.MagicMock(return_value=mock_path)) mock_path.__truediv__.return_value = mock_path mock_path.exists.return_value = True mock_json = mocker.MagicMock() - monkeypatch.setattr(implementation, "json", mock_json) + monkeypatch.setattr(module, "json", mock_json) mock_json.load.return_value = {"run_id": "test"} - run_log_store = implementation.FileSystemRunLogstore() + run_log_store = module.FileSystemRunLogstore() run_log = run_log_store.get_from_folder(run_id="does not matter") assert run_log.run_id == "test" @@ -85,10 +78,10 @@ def test_file_system_run_log_store_create_run_log_writes_to_folder(mocker, monke mock_write_to_folder = mocker.MagicMock() monkeypatch.setattr( - implementation.FileSystemRunLogstore, "write_to_folder", mock_write_to_folder + module.FileSystemRunLogstore, "write_to_folder", mock_write_to_folder ) - run_log_store = implementation.FileSystemRunLogstore() + run_log_store = module.FileSystemRunLogstore() run_log = run_log_store.create_run_log(run_id="test random") mock_write_to_folder.assert_called_once_with(run_log) @@ -103,15 +96,15 @@ def test_file_system_run_log_store_create_run_log_raises_exception_if_present( mock_get_run_log_by_id = mocker.MagicMock(return_value="existing") monkeypatch.setattr( - implementation.FileSystemRunLogstore, "write_to_folder", mock_write_to_folder + module.FileSystemRunLogstore, "write_to_folder", mock_write_to_folder ) monkeypatch.setattr( - implementation.FileSystemRunLogstore, + module.FileSystemRunLogstore, "get_run_log_by_id", mock_get_run_log_by_id, ) - run_log_store = implementation.FileSystemRunLogstore() + run_log_store = module.FileSystemRunLogstore() with pytest.raises(exceptions.RunLogExistsError): run_log_store.create_run_log(run_id="test random") @@ -123,10 +116,10 @@ def test_file_system_run_log_store_get_run_log_by_id_raises_exception_if_get_fro mock_get_from_folder.side_effect = FileNotFoundError() monkeypatch.setattr( - implementation.FileSystemRunLogstore, "get_from_folder", mock_get_from_folder + module.FileSystemRunLogstore, "get_from_folder", mock_get_from_folder ) - run_log_store = implementation.FileSystemRunLogstore() + run_log_store = module.FileSystemRunLogstore() with pytest.raises(exceptions.RunLogNotFoundError): run_log_store.get_run_log_by_id(run_id="should fail") @@ -138,10 +131,10 @@ def test_file_system_run_log_store_get_run_log_by_id_returns_run_log_from_get_fr mock_get_from_folder.return_value = "I am a run log" monkeypatch.setattr( - implementation.FileSystemRunLogstore, "get_from_folder", mock_get_from_folder + module.FileSystemRunLogstore, "get_from_folder", mock_get_from_folder ) - run_log_store = implementation.FileSystemRunLogstore() + run_log_store = module.FileSystemRunLogstore() run_log = run_log_store.get_run_log_by_id(run_id="test") @@ -152,10 +145,10 @@ def test_file_system_run_log_store_put_run_log_writes_to_folder(mocker, monkeypa mock_write_to_folder = mocker.MagicMock() monkeypatch.setattr( - implementation.FileSystemRunLogstore, "write_to_folder", mock_write_to_folder + module.FileSystemRunLogstore, "write_to_folder", mock_write_to_folder ) - run_log_store = implementation.FileSystemRunLogstore() + run_log_store = module.FileSystemRunLogstore() mock_run_log = mocker.MagicMock() run_log_store.put_run_log(run_log=mock_run_log) diff --git a/tests/extensions/run_log_store/test_generic_chunked.py b/tests/extensions/run_log_store/test_generic_chunked.py deleted file mode 100644 index 9b003829..00000000 --- a/tests/extensions/run_log_store/test_generic_chunked.py +++ /dev/null @@ -1 +0,0 @@ -from runnable.extensions.run_log_store import generic_chunked diff --git a/tests/extensions/run_log_store/test_k8s_pvc_integration.py b/tests/extensions/run_log_store/test_k8s_pvc_integration.py deleted file mode 100644 index 3daf7093..00000000 --- a/tests/extensions/run_log_store/test_k8s_pvc_integration.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from runnable.extensions.run_log_store.k8s_pvc import integration - - -def test_k8s_pvc_errors_for_local(): - test_integration = integration.LocalCompute( - executor="executor", integration_service="catalog" - ) - - with pytest.raises(Exception, match="We can't use the local compute"): - test_integration.validate() - - -def test_k8s_pvc_errors_for_local_container(): - test_integration = integration.LocalContainerCompute( - executor="executor", integration_service="catalog" - ) - - with pytest.raises(Exception, match="We can't use the local-container compute"): - test_integration.validate() diff --git a/tests/extensions/secrets/test_dotenv.py b/tests/extensions/secrets/test_dotenv.py index e12677dd..1d41db6c 100644 --- a/tests/extensions/secrets/test_dotenv.py +++ b/tests/extensions/secrets/test_dotenv.py @@ -1,9 +1,5 @@ -import pytest - -from runnable import defaults, exceptions - -from runnable.extensions.secrets.dotenv.implementation import DotEnvSecrets -import runnable.extensions.secrets.dotenv.implementation as implementation +from extensions.secrets.dotenv import DotEnvSecrets +from runnable import defaults def test_dot_env_secrets_defaults_to_default_location_if_none_provided( @@ -18,75 +14,4 @@ def test_dot_env_secrets_usees_location_if_provided(mocker, monkeypatch): assert dot_env_secret.location == "here" -def test_dot_env_secrets_get_returns_secret_if_one_provided(mocker, monkeypatch): - dot_env_secret = DotEnvSecrets(location="here") - dot_env_secret.secrets["give"] = "this" - - assert dot_env_secret.get("give") == "this" - - -def test_dot_env_secrets_raises_exception_if_secret_not_found(mocker, monkeypatch): - monkeypatch.setattr(DotEnvSecrets, "_load_secrets", mocker.MagicMock()) - - dot_env_secret = DotEnvSecrets(location="here") - dot_env_secret.secrets["give"] = "this" - - with pytest.raises(expected_exception=exceptions.SecretNotFoundError): - dot_env_secret.get("give1") - - -def test_dot_env_load_secrets_raises_exception_if_file_does_not_exist( - mocker, monkeypatch -): - monkeypatch.setattr( - implementation.utils, "does_file_exist", mocker.MagicMock(return_value=False) - ) - - dot_env_secret = DotEnvSecrets(location="here") - - with pytest.raises(Exception, match="Did not find the secrets file"): - dot_env_secret._load_secrets() - - -def test_dot_env_load_secrets_raises_exception_if_secret_formatting_is_invalid( - mocker, monkeypatch -): - monkeypatch.setattr( - implementation.utils, "does_file_exist", mocker.MagicMock(return_value=True) - ) - - dot_env_secret = DotEnvSecrets(location="here") - - with pytest.raises( - Exception, match="A secret should be of format, secret_name=secret_value" - ): - mocker.patch("builtins.open", mocker.mock_open(read_data="data")) - dot_env_secret._load_secrets() - - -def test_dot_env_load_secrets_raises_exception_if_secret_formatting_is_invalid_ge_2( - mocker, monkeypatch -): - monkeypatch.setattr( - implementation.utils, "does_file_exist", mocker.MagicMock(return_value=True) - ) - - dot_env_secret = DotEnvSecrets(location="here") - - with pytest.raises( - Exception, match="A secret should be of format, secret_name=secret_value" - ): - mocker.patch("builtins.open", mocker.mock_open(read_data="data=data1=")) - dot_env_secret._load_secrets() - - -def test_dot_env_load_secrets_populates_correct_secrets_if_valid(mocker, monkeypatch): - monkeypatch.setattr( - implementation.utils, "does_file_exist", mocker.MagicMock(return_value=True) - ) - - dot_env_secret = DotEnvSecrets(location="here") - - mocker.patch("builtins.open", mocker.mock_open(read_data="data=data1\n")) - dot_env_secret._load_secrets() - assert dot_env_secret.secrets == {"data": "data1"} +# TODO: dotenv testing diff --git a/tests/extensions/test_node_extensions.py b/tests/extensions/test_node_extensions.py index 440f7190..29fe2993 100644 --- a/tests/extensions/test_node_extensions.py +++ b/tests/extensions/test_node_extensions.py @@ -1,7 +1,6 @@ import pytest from runnable import defaults -from runnable.extensions import nodes as nodes from runnable.tasks import BaseTaskType @@ -261,3 +260,6 @@ def test__as_is_node_takes_anything_as_input(mocker, monkeypatch): } _ = nodes.StubNode.parse_from_config(config=config) + + +from extensions.nodes import nodes