From 69acb62d1c0387bd8ed08851ffefa7b032f12406 Mon Sep 17 00:00:00 2001 From: Giuseppe Steduto Date: Wed, 18 Oct 2023 16:27:38 +0200 Subject: [PATCH] api: fix create_workflow_from_json to load workflow specification Amend `create_workflow_from_json` so that the workflow specification is always loaded in the REANA specification that is passed to reana-server. Closes #666. --- CHANGES.rst | 1 + reana_client/api/client.py | 30 ++++++++++++++++------- setup.py | 2 +- tests/conftest.py | 45 +++++++++++++++++++++++++++++++++++ tests/test_cli_workflows.py | 47 +++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 9 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 6df40c95..749ebc2e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,7 @@ Changes Version 0.9.2 (UNRELEASED) -------------------------- +- Fixes ``create_workflow_from_json`` API command to always send the workflow specification to the server. - Fixes ``list`` command to be case-insensitive when using the ``--sort`` flag to sort the workflow runs by a specific column name. Version 0.9.1 (2023-09-27) diff --git a/reana_client/api/client.py b/reana_client/api/client.py index 42c25444..ba0d83bb 100644 --- a/reana_client/api/client.py +++ b/reana_client/api/client.py @@ -18,6 +18,10 @@ import requests from bravado.exception import HTTPError from reana_commons.validation.utils import validate_reana_yaml, validate_workflow_name +from reana_commons.specification import ( + load_workflow_spec_from_reana_yaml, + load_input_parameters, +) from reana_commons.api_client import get_current_api_client from reana_commons.config import REANA_WORKFLOW_ENGINES from reana_commons.errors import REANASecretAlreadyExists, REANASecretDoesNotExist @@ -248,6 +252,7 @@ def create_workflow_from_json( parameters=None, workflow_engine="yadage", outputs=None, + workspace_path=None, ): """Create a workflow from JSON specification. @@ -259,6 +264,7 @@ def create_workflow_from_json( :param parameters: workflow input parameters dictionary. :param workflow_engine: one of the workflow engines (yadage, serial, cwl) :param outputs: dictionary with expected workflow outputs. + :param workspace_path: path to the workspace where the workflow is located. :return: if the workflow was created successfully, a dictionary with the information about the ``workflow_id`` and ``workflow_name``, along with a ``message`` of success. @@ -290,21 +296,29 @@ def create_workflow_from_json( ) try: reana_yaml = dict(workflow={}) - if workflow_file: - reana_yaml["workflow"]["file"] = workflow_file - else: - reana_yaml["workflow"]["specification"] = workflow_json reana_yaml["workflow"]["type"] = workflow_engine if parameters: reana_yaml["inputs"] = parameters if outputs: reana_yaml["outputs"] = outputs + if workflow_file: + reana_yaml["workflow"]["file"] = workflow_file + reana_yaml["workflow"][ + "specification" + ] = load_workflow_spec_from_reana_yaml(reana_yaml, workspace_path) + else: + reana_yaml["workflow"]["specification"] = workflow_json + # The function below loads the input parameters into the reana_yaml dictionary + # taking them from the parameters yaml files (used by CWL and Snakemake workflows), + # and replacing the `input.parameters.input` field with the actual parameters values. + # For this reason, we have to load the workflow specification first, as otherwise + # the specification validation would fail. + input_params = load_input_parameters(reana_yaml, workspace_path) + if input_params is not None: + reana_yaml["inputs"]["parameters"] = input_params validate_reana_yaml(reana_yaml) - reana_specification = reana_yaml (response, http_response) = current_rs_api_client.api.create_workflow( - reana_specification=json.loads( - json.dumps(reana_specification, sort_keys=True) - ), + reana_specification=json.loads(json.dumps(reana_yaml, sort_keys=True)), workflow_name=name, access_token=access_token, ).result() diff --git a/setup.py b/setup.py index 9096ef69..60a86067 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ "click>=7", "pathspec==0.9.0", "jsonpointer>=2.0", - "reana-commons[yadage,snakemake,cwl]>=0.9.3,<0.10.0", + "reana-commons[yadage,snakemake,cwl]>=0.9.4a1,<0.10.0", "tablib>=0.12.1,<0.13", "werkzeug>=0.14.1 ; python_version<'3.10'", "werkzeug>=0.15.0 ; python_version>='3.10'", diff --git a/tests/conftest.py b/tests/conftest.py index 369beb3d..b6d483c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ from __future__ import absolute_import, print_function +import textwrap + import pytest from typing import Dict @@ -119,6 +121,49 @@ def cwl_workflow_spec_step(): return cwl_workflow_spec_step +@pytest.fixture() +def create_snakemake_yaml_external_input_workflow_schema(): + """Return dummy schema for a Snakemake workflow with external parameters.""" + reana_cwl_yaml_schema = """ + inputs: + parameters: + input: config.yaml + workflow: + type: snakemake + file: Snakefile + outputs: + files: + - foo.txt + """ + return reana_cwl_yaml_schema + + +@pytest.fixture() +def snakemake_workflow_spec_step_param(): + """Return dummy Snakemake workflow loaded spec.""" + snakefile = textwrap.dedent( + """ + rule foo: + params: + param1=config["param1"], + param2=config["param2"], + output: + "foo.txt" + """ + ) + return snakefile + + +@pytest.fixture() +def external_parameter_yaml_file(): + """Return dummy external parameter YAML file.""" + config_yaml = """ + param1: 200 + param2: 300 + """ + return config_yaml + + @pytest.fixture() def cwl_workflow_spec_correct_input_param(): """Return correct dummy CWL workflow loaded spec.""" diff --git a/tests/test_cli_workflows.py b/tests/test_cli_workflows.py index 624b8626..4f6bc82c 100644 --- a/tests/test_cli_workflows.py +++ b/tests/test_cli_workflows.py @@ -9,6 +9,7 @@ """REANA client workflow tests.""" import json +import sys from typing import List import pytest @@ -572,6 +573,52 @@ def test_create_workflow_from_json(create_yaml_workflow_schema): assert response["message"] == result["message"] +def test_create_snakemake_workflow_from_json_parameters( + create_snakemake_yaml_external_input_workflow_schema, + tmp_path, + snakemake_workflow_spec_step_param, + external_parameter_yaml_file, +): + """Test create workflow from json with external parameters.""" + if sys.version_info.major == 3 and sys.version_info.minor in (11, 12): + pytest.xfail( + "Snakemake features of reana-client are not supported on Python 3.11" + ) + status_code = 201 + response = { + "message": "The workflow has been successfully created.", + "workflow_id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac", + "workflow_name": "mytest", + } + env = {"REANA_SERVER_URL": "localhost"} + reana_token = "000000" + mock_http_response, mock_response = Mock(), Mock() + mock_http_response.status_code = status_code + mock_response = response + workflow_json = yaml.load( + create_snakemake_yaml_external_input_workflow_schema, Loader=yaml.FullLoader + ) + with open(tmp_path / "Snakefile", "w") as f: + f.write(snakemake_workflow_spec_step_param) + with open(tmp_path / "config.yaml", "w") as f: + f.write(external_parameter_yaml_file) + with patch.dict("os.environ", env): + with patch( + "reana_client.api.client.current_rs_api_client", + make_mock_api_client("reana-server")(mock_response, mock_http_response), + ): + result = create_workflow_from_json( + workflow_file=str(tmp_path / "Snakefile"), + name=response["workflow_name"], + access_token=reana_token, + parameters=workflow_json["inputs"], + workflow_engine="snakemake", + workspace_path=str(tmp_path), + ) + assert response["workflow_name"] == result["workflow_name"] + assert response["message"] == result["message"] + + @pytest.mark.parametrize( "status, exit_code", [