From 4d90b2f38c7fc0b75b9ec1b7fc55788d2dc6e242 Mon Sep 17 00:00:00 2001
From: Marco Donadoni <marco.donadoni@cern.ch>
Date: Fri, 23 Feb 2024 14:05:19 +0100
Subject: [PATCH] feat(manager): increase termination period of run-batch pods
 (#572)

---
 reana_workflow_controller/config.py               | 8 ++++++++
 reana_workflow_controller/workflow_run_manager.py | 2 ++
 2 files changed, 10 insertions(+)

diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py
index fb3cff0c..e94b3033 100644
--- a/reana_workflow_controller/config.py
+++ b/reana_workflow_controller/config.py
@@ -175,3 +175,11 @@
     RunStatus.pending,
 ]
 """Alive workflow statuses."""
+
+REANA_RUNTIME_BATCH_TERMINATION_GRACE_PERIOD = int(
+    os.getenv("REANA_RUNTIME_BATCH_TERMINATION_GRACE_PERIOD", "120")
+)
+"""Grace period before terminating the job controller and workflow engine pod.
+
+The job controller needs to clean up all the running jobs before the end of the grace period.
+"""
diff --git a/reana_workflow_controller/workflow_run_manager.py b/reana_workflow_controller/workflow_run_manager.py
index 92589093..e8835085 100644
--- a/reana_workflow_controller/workflow_run_manager.py
+++ b/reana_workflow_controller/workflow_run_manager.py
@@ -61,6 +61,7 @@
     IMAGE_PULL_SECRETS,
     JOB_CONTROLLER_CONTAINER_PORT,
     JOB_CONTROLLER_SHUTDOWN_ENDPOINT,
+    REANA_RUNTIME_BATCH_TERMINATION_GRACE_PERIOD,
     REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT,
     REANA_KUBERNETES_JOBS_MEMORY_LIMIT,
     REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT,
@@ -652,6 +653,7 @@ def _create_job_spec(
             containers=containers,
             node_selector=REANA_RUNTIME_BATCH_KUBERNETES_NODE_LABEL,
             init_containers=[],
+            termination_grace_period_seconds=REANA_RUNTIME_BATCH_TERMINATION_GRACE_PERIOD,
         )
         spec.template.spec.service_account_name = (
             REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME