add tests for new CWL loading feature

CHANGES.rst

@@ -32,6 +32,8 @@ Changes:
   known `Process` definitions from `CWL` files stored in a nested directory structure. This allows a service provider
   that uses `Weaver` to offer their `Processes` to directly maintain their definitions from the set of `CWL` files and
   upload changes in the web application at startup without need to manually undeploy and redeploy each `Process`.
+- Add ``weaver.cwl_processes_register_error`` to fail fast any `Process` registration error from `CWL` when loading
+  files at startup.
 
 Fixes:
 ------

config/weaver.ini.example

@@ -85,6 +85,7 @@ weaver.cwl_egid =
 # default configuration directory is used if this entry is removed
 # only CWL files are considered, lookup in directory is recursive
 weaver.cwl_processes_dir =
+weaver.cwl_processes_register_error = false
 
 # --- Weaver WPS settings ---
 weaver.wps = true

docs/source/configuration.rst

@@ -402,12 +402,17 @@ in an identical definition as if it was :ref:`Deployed <proc_op_deploy>` using :
 - | ``weaver.cwl_processes_dir = <dir-path>``
   | (default: :py:data:`WEAVER_CONFIG_DIR`)
   |
-  | Defines the root directory where to *recursively* and *alphabetically* (as flat list) load any :term:`CWL` file
-    to deploy the corresponding :term:`Process` definitions.
+  | Defines the root directory where to *recursively* and *alphabetically* load any :term:`CWL` file
+    to deploy the corresponding :term:`Process` definitions. Files at higher levels are loaded first before moving
+    down into lower directories of the structure.
+  |
+  | Any failed deployment from a seemingly valid :term:`CWL` will be logged with the corresponding error message.
+    Loading will proceed by ignoring failing cases according to ``weaver.cwl_processes_register_error`` setting.
+    The number of successful :term:`Process` deployments will also be reported if any should occur.
   |
   | The value defined by this setting will look for the provided path as absolute location, then will attempt to
-    resolve relative path (corresponding to where the application is started from). If none of the files can be found,
-    the operation is skipped.
+    resolve relative path (corresponding to where the application is started from). If no :term:`CWL` file could be
+    found, the operation is skipped.
   |
   | To ensure that this feature is disabled and to avoid any unexpected auto-deployment provided by this functionality,
     simply set setting ``weaver.cwl_processes_dir`` as *undefined* (i.e.: nothing after ``=`` in ``weaver.ini``).
@@ -422,6 +427,13 @@ in an identical definition as if it was :ref:`Deployed <proc_op_deploy>` using :
     dependencies must be registered prior to this :term:`Process`. Consider naming your :term:`CWL` files to take
     advantage of loading order to resolve such situations.
 
+- | ``weaver.cwl_processes_register_error = true|false`` [:class:`bool`]
+  | (default: ``false``, *ignore failures*)
+  |
+  | Indicate if `Weaver` should ignore failing :term:`Process` deployments (when ``false``), due to unsuccessful
+    registration of :term:`CWL` files found within any sub-directory of ``weaver.cwl_processes_dir`` path, or
+    immediately fail (when ``true``) when an issue is raised during :term:`Process` deployment.
+
 .. seealso::
     - `weaver.ini.example`_
 

tests/processes/test_utils.py

@@ -1,5 +1,8 @@
+import json
+import os
 import tempfile
 
+import mock
 import pytest
 import yaml
 
@@ -10,8 +13,9 @@
     setup_mongodb_processstore,
     setup_mongodb_servicestore
 )
+from weaver.exceptions import PackageRegistrationError
 from weaver.processes.constants import CWL_REQUIREMENT_APP_WPS1
-from weaver.processes.utils import register_wps_processes_from_config
+from weaver.processes.utils import register_cwl_processes_from_config, register_wps_processes_from_config
 
 WPS1_URL1 = resources.TEST_REMOTE_SERVER_URL
 WPS1_URL2 = "http://yet-another-server.com"
@@ -186,3 +190,144 @@ def test_register_wps_processes_from_config_valid():
     proc6 = p_store.fetch_by_id(proc6_id)
     assert proc6.package["hints"][CWL_REQUIREMENT_APP_WPS1]["provider"] == WPS1_URL4 + "/"
     assert proc6.package["hints"][CWL_REQUIREMENT_APP_WPS1]["process"] == resources.WPS_LITERAL_COMPLEX_IO_ID
+
+
+def test_register_cwl_processes_from_config_undefined():
+    assert register_cwl_processes_from_config({}) == 0
+
+
+def test_register_cwl_processes_from_config_empty_var():
+    settings = {"weaver.cwl_processes_dir": ""}
+    assert register_cwl_processes_from_config(settings) == 0
+
+
+def test_register_cwl_processes_from_config_not_a_dir():
+    with tempfile.NamedTemporaryFile(mode="w") as tmp_file:
+        tmp_file.write("data")
+
+        settings = {"weaver.cwl_processes_dir": tmp_file.name}
+        assert register_cwl_processes_from_config(settings) == 0
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_dir = os.path.join(tmp_dir, "does-not-exist")
+        settings = {"weaver.cwl_processes_dir": tmp_dir}
+        assert register_cwl_processes_from_config(settings) == 0
+
+
+def test_register_cwl_processes_from_config_dir_no_cwl():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        settings = {"weaver.cwl_processes_dir": tmp_dir}
+        assert register_cwl_processes_from_config(settings) == 0
+
+        with tempfile.NamedTemporaryFile(dir=tmp_dir, suffix=".json", mode="w", delete=False) as tmp_file:
+            tmp_file.write(json.dumps({"data": "test"}))
+
+        assert register_cwl_processes_from_config(settings) == 0
+
+
+def test_register_cwl_processes_from_config_load_recursive():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        first_dir = os.path.join(tmp_dir, "first")
+        nested_dir = os.path.join(tmp_dir, "nested")
+        deeper_dir = os.path.join(nested_dir, "deeper")
+        os.makedirs(first_dir)
+        os.makedirs(deeper_dir)
+
+        # Write files in **un**ordered fashion to validate ordered loading occurs:
+        # /tmp
+        #   /dir
+        #     file3.cwl
+        #     random.yml
+        #     file5.cwl
+        #     /first
+        #       b_file9.cwl         # note: must appear before file2 and a_file8, 'nested' loaded after 'first'
+        #       file2.cwl
+        #       invalid.cwl
+        #     /nested
+        #       a_file8.cwl         # note: must appear after file2 and b_file9, 'nested' loaded after 'first'
+        #       random.json
+        #       file1.cwl
+        #       file4.cwl
+        #       /deeper
+        #         c_file7.cwl
+        #         file0.cwl
+        #         file6.cwl
+        #         invalid.cwl
+        #
+        # Loaded order:
+        #   /tmp/dir/file3.cwl
+        #   /tmp/dir/file5.cwl
+        #   /tmp/dir/first/b_file9.cwl
+        #   /tmp/dir/first/file2.cwl
+        #   /tmp/dir/nested/a_file8.cwl
+        #   /tmp/dir/nested/file1.cwl               # note:
+        #   /tmp/dir/nested/file4.cwl               dir 'deeper' purposely named to appear before
+        #   /tmp/dir/nested/deeper/file0.cwl        'file#' one level above if they were sorted *only*
+        #   /tmp/dir/nested/deeper/file6.cwl        alphabetically not considering directory structure
+        valid_order = [3, 5, 9, 2, 8, 1, 4, 7, 0, 6]
+        # doest not need to be valid CWL, mocked loading
+        cwl_ordered = [{"cwlVersion": "v1.0", "id": str(i)} for i in range(len(valid_order))]
+        with open(os.path.join(tmp_dir, "file3.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[3]))
+        with open(os.path.join(tmp_dir, "file5.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[5]))
+        with open(os.path.join(tmp_dir, "random.yml"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write("random: data")
+        with open(os.path.join(first_dir, "file2.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[2]))
+        with open(os.path.join(first_dir, "invalid.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps({"invalid": True}))
+        with open(os.path.join(first_dir, "b_file9.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[9]))
+        with open(os.path.join(nested_dir, "file1.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[1]))
+        with open(os.path.join(nested_dir, "file4.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[4]))
+        with open(os.path.join(nested_dir, "random.json"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps({"random": "data"}))
+        with open(os.path.join(nested_dir, "a_file8.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[8]))
+        with open(os.path.join(deeper_dir, "c_file7.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[7]))
+        with open(os.path.join(deeper_dir, "file0.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[0]))
+        with open(os.path.join(deeper_dir, "file6.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps(cwl_ordered[6]))
+        with open(os.path.join(deeper_dir, "invalid.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write(json.dumps({"invalid": True}))
+
+        def no_op_valid(_cwl, *_, **__):  # type: ignore
+            if isinstance(_cwl, dict) and "invalid" in _cwl:
+                raise PackageRegistrationError("CWL INVALID")
+
+        with mock.patch("weaver.processes.utils.deploy_process_from_payload", side_effect=no_op_valid) as mocked:
+            settings = {"weaver.cwl_processes_dir": tmp_dir}
+            assert register_cwl_processes_from_config(settings) == len(cwl_ordered)
+
+        call_count = len(cwl_ordered) + 2  # 2 invalid
+        assert mocked.call_count == call_count
+        valid_calls = list(call for call in mocked.call_args_list if "invalid" not in call.args[0])
+        assert len(valid_calls) == len(cwl_ordered)
+        for i, (order, call) in enumerate(zip(valid_order, valid_calls)):
+            assert call.args[0] == cwl_ordered[order], f"Expected CWL does not match load order at position: {i}"
+
+
+def test_register_cwl_processes_from_config_error_handling():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        with open(os.path.join(tmp_dir, "ignore.cwl"), mode="w", encoding="utf-8") as tmp_file:
+            tmp_file.write("not important")
+
+        def raise_deploy(*_, **__):
+            raise PackageRegistrationError("test")
+
+        with mock.patch("weaver.processes.utils.deploy_process_from_payload", side_effect=raise_deploy) as mocked:
+            settings = {"weaver.cwl_processes_dir": tmp_dir}
+            assert register_cwl_processes_from_config(settings) == 0
+            assert mocked.call_count == 1
+        with mock.patch("weaver.processes.utils.deploy_process_from_payload", side_effect=raise_deploy) as mocked:
+            result = None
+            with pytest.raises(PackageRegistrationError) as exc:
+                settings["weaver.cwl_processes_register_error"] = "true"
+                result = register_cwl_processes_from_config(settings)
+            assert mocked.call_count == 1
+            assert result is None  # not returned

weaver/database/__init__.py

@@ -40,7 +40,7 @@ def get_db(container=None, reset_connection=False):
 def includeme(config):
     # type: (Configurator) -> None
     settings = get_settings(config)
-    if asbool(settings.get("weaver.build_docs", False)):
+    if asbool(settings.get("weaver.build_docs", False)):  # pragma: no cover
         LOGGER.info("Skipping database when building docs...")
         return
 

weaver/exceptions.py

@@ -412,6 +412,7 @@ def handle_known_exceptions(function):
 
     @functools.wraps(function)
     def wrapped(*_, **__):
+        # type: (Any, Any) -> Any
         try:
             return function(*_, **__)
         except (WeaverException, OWSException, HTTPException) as exc:
@@ -466,6 +467,7 @@ def wrap(function):
         # type: (Callable[[Any, Any], Any]) -> Callable
         @functools.wraps(function)
         def call(*args, **kwargs):
+            # type: (Any, Any) -> Any
             try:
                 # handle input arguments that are extended by various pyramid operations
                 if is_request:

weaver/processes/utils.py

@@ -692,7 +692,7 @@ def register_wps_processes_from_config(container, wps_processes_file_path=None):
 
 
 def register_cwl_processes_from_config(container):
-    # type: (AnyRegistryContainer) -> int
+    # type: (AnySettingsContainer) -> int
     """
     Load multiple :term:`CWL` definitions from a directory to register corresponding :term:`Process`.
 
@@ -737,7 +737,9 @@ def register_cwl_processes_from_config(container):
         )
         return 0
     cwl_processes_dir = os.path.abspath(cwl_processes_dir)
-    cwl_files = sorted(pathlib.Path(cwl_processes_dir).rglob("*.cwl"))
+    cwl_files = sorted(pathlib.Path(cwl_processes_dir).rglob("*.cwl"),
+                       # consider directory structure to sort, then use usual alphabetical order for same level
+                       key=lambda file: (len(str(file).split("/")), str(file)))
     if not cwl_files:
         warnings.warn(
             f"Configuration directory [{cwl_processes_dir}] for CWL processes registration "
@@ -747,16 +749,22 @@ def register_cwl_processes_from_config(container):
 
     register_count = 0
     register_total = len(cwl_files)
+    register_error = asbool(settings.get("weaver.cwl_processes_register_error", False))
     for cwl_path in cwl_files:
         try:
             cwl = load_package_file(str(cwl_path))
             deploy_process_from_payload(cwl, settings, overwrite=True)
+            register_count += 1
         except (HTTPException, PackageRegistrationError) as exc:
-            warnings.warn(
+            msg = (
                 f"Failed registration of process from CWL file: [{cwl_path!s}] "
-                f"caused by [{fully_qualified_name(exc)}]({exc!s}). "
-                "Skipping definition.", RuntimeWarning
+                f"caused by [{fully_qualified_name(exc)}]({exc!s})."
             )
+            if register_error:
+                LOGGER.info("Requested immediate CWL registration failure with 'weaver.cwl_processes_register_error'.")
+                LOGGER.error(msg)
+                raise
+            warnings.warn(msg + " Skipping definition.", RuntimeWarning)
             continue
     if register_count and register_count == register_total:
         LOGGER.info("Successfully registered %s processes from CWL files.", register_total)

weaver/typedefs.py

@@ -1,4 +1,4 @@
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING  # pragma: no cover
 
 if TYPE_CHECKING:
     import os