diff --git a/CHANGES.rst b/CHANGES.rst index dadd73ea..d97adc27 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,7 @@ Version 0.9.1 (UNRELEASED) - Changes uWSGI configuration to increase buffer size, add vacuum option, etc. - Fixes job status inconsistency by correctly setting running jobs' status to ``stopped`` when a workflow is stopped. - Fixes uWSGI memory consumption on systems with very high allowed number of open files. +- Fixes uWSGI and ``consume-job-queue`` command to gracefully stop when being terminated. Version 0.9.0 (2023-01-19) -------------------------- diff --git a/Dockerfile b/Dockerfile index b9ee743c..f9b00f6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,10 +66,14 @@ ENV FLASK_APP=reana_workflow_controller/app.py \ EXPOSE 5000 # Run server +# exec is used to make sure signals are propagated to uwsgi, +# while also allowing shell expansion # hadolint ignore=DL3025 -CMD uwsgi \ +CMD exec uwsgi \ --buffer-size ${UWSGI_BUFFER_SIZE} \ --die-on-term \ + --hook-master-start "unix_signal:2 gracefully_kill_them_all" \ + --hook-master-start "unix_signal:15 gracefully_kill_them_all" \ --enable-threads \ --http-socket 0.0.0.0:5000 \ --master \ diff --git a/reana_workflow_controller/cli.py b/reana_workflow_controller/cli.py index 2a80b37e..8edca520 100644 --- a/reana_workflow_controller/cli.py +++ b/reana_workflow_controller/cli.py @@ -9,6 +9,7 @@ """REANA Workflow Controller command line interface.""" import logging +import signal import click from reana_commons.config import REANA_LOG_FORMAT, REANA_LOG_LEVEL @@ -21,4 +22,12 @@ def consume_job_queue(): """Consumes job queue and updates job status.""" logging.basicConfig(level=REANA_LOG_LEVEL, format=REANA_LOG_FORMAT) consumer = JobStatusConsumer() + + def stop_consumer(signum, frame): + logging.info("Stopping job status consumer...") + consumer.should_stop = True + + signal.signal(signal.SIGTERM, stop_consumer) + + logging.info("Starting job status consumer...") consumer.run()