feat(manager): pass custom env variables to workflow engines (reanahu…

…b#571)
mdonadoni · Feb 22, 2024 · 56b00f5 · 56b00f5
1 parent 08ab9a3
commit 56b00f5
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 5 deletions.
diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py
@@ -117,6 +117,28 @@
 ]
 """Common to all workflow engines environment variables."""
 
+
+def _env_vars_dict_to_k8s_list(env_vars):
+    return [{"name": name, "value": value} for name, value in env_vars.items()]
+
+
+WORKFLOW_ENGINE_CWL_ENV_VARS = _env_vars_dict_to_k8s_list(
+    json.loads(os.getenv("REANA_WORKFLOW_ENGINE_CWL_ENV_VARS", "{}"))
+)
+"""Environment variables to be passed to the CWL workflow engine container."""
+WORKFLOW_ENGINE_SERIAL_ENV_VARS = _env_vars_dict_to_k8s_list(
+    json.loads(os.getenv("REANA_WORKFLOW_ENGINE_SERIAL_ENV_VARS", "{}"))
+)
+"""Environment variables to be passed to the serial workflow engine container."""
+WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS = _env_vars_dict_to_k8s_list(
+    json.loads(os.getenv("REANA_WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS", "{}"))
+)
+"""Environment variables to be passed to the Snakemake workflow engine container."""
+WORKFLOW_ENGINE_YADAGE_ENV_VARS = _env_vars_dict_to_k8s_list(
+    json.loads(os.getenv("REANA_WORKFLOW_ENGINE_YADAGE_ENV_VARS", "{}"))
+)
+"""Environment variables to be passed to the Yadage workflow engine container."""
+
 DEBUG_ENV_VARS = (
     {
         "name": "WDB_SOCKET_SERVER",

diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py
@@ -6,6 +6,7 @@
 
 """Workflow run manager interface."""
 import base64
+import copy
 import json
 import logging
 import os
@@ -71,6 +72,10 @@
     REANA_WORKFLOW_ENGINE_IMAGE_YADAGE,
     WORKFLOW_ENGINE_COMMON_ENV_VARS,
     DEBUG_ENV_VARS,
+    WORKFLOW_ENGINE_CWL_ENV_VARS,
+    WORKFLOW_ENGINE_SERIAL_ENV_VARS,
+    WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS,
+    WORKFLOW_ENGINE_YADAGE_ENV_VARS,
 )
 
 
@@ -92,7 +97,8 @@ class WorkflowRunManager:
                 "--workflow-parameters '{parameters}' "
                 "--operational-options '{options}' "
             ),
-            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
+            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+            + WORKFLOW_ENGINE_CWL_ENV_VARS,
         },
         "yadage": {
             "image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_YADAGE),
@@ -105,7 +111,8 @@ class WorkflowRunManager:
                 "--workflow-parameters '{parameters}' "
                 "--operational-options '{options}' "
             ),
-            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
+            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+            + WORKFLOW_ENGINE_YADAGE_ENV_VARS,
         },
         "serial": {
             "image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_SERIAL),
@@ -117,7 +124,8 @@ class WorkflowRunManager:
                 "--workflow-parameters '{parameters}' "
                 "--operational-options '{options}' "
             ),
-            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
+            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+            + WORKFLOW_ENGINE_SERIAL_ENV_VARS,
         },
         "snakemake": {
             "image": "{}".format(REANA_WORKFLOW_ENGINE_IMAGE_SNAKEMAKE),
@@ -129,7 +137,8 @@ class WorkflowRunManager:
                 "--workflow-parameters '{parameters}' "
                 "--operational-options '{options}' "
             ),
-            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS,
+            "environment_variables": WORKFLOW_ENGINE_COMMON_ENV_VARS
+            + WORKFLOW_ENGINE_SNAKEMAKE_ENV_VARS,
         },
     }
     """Mapping between engines and their basis configuration."""
@@ -220,7 +229,7 @@ def retrieve_required_cvmfs_repos(self):
 
     def _workflow_engine_env_vars(self):
         """Return necessary environment variables for the workflow engine."""
-        env_vars = list(
+        env_vars = copy.deepcopy(
             WorkflowRunManager.engine_mapping[self.workflow.type_][
                 "environment_variables"
             ]