From ea374260e345e7ec1056c7366dbd8a864433e764 Mon Sep 17 00:00:00 2001 From: Jason Joyce Date: Thu, 28 Mar 2024 12:24:53 -0400 Subject: [PATCH] Adding celery scaffold to the project. Adding celery_scaffold for use with flask based projects. It provides a way to configure celery for use, using the flask configuration files. It also provides a celery_app and a flask_app that can be used in your project. A base_scaffold was pulled out due to the celery worker assuming any 'app' attribute is of type Celery. The original app_scaffold is available to provide backward compatibility. It leverages the new base_scaffold and sets the 'app' attribute to flask_app to ensure existing use cases are handled. Signed-off-by: Jason Joyce --- Pipfile | 2 + README.md | 45 ++++++++ setup.cfg | 3 + src/flask_container_scaffold/app_scaffold.py | 96 ++--------------- src/flask_container_scaffold/base_scaffold.py | 102 ++++++++++++++++++ .../celery_scaffold.py | 30 ++++++ test-requirements.txt | 2 +- tests/unit/test_celery.py | 65 +++++++++++ 8 files changed, 256 insertions(+), 89 deletions(-) create mode 100644 src/flask_container_scaffold/base_scaffold.py create mode 100644 src/flask_container_scaffold/celery_scaffold.py create mode 100644 tests/unit/test_celery.py diff --git a/Pipfile b/Pipfile index 85e5967..12d0791 100644 --- a/Pipfile +++ b/Pipfile @@ -9,3 +9,5 @@ flask-container-scaffold = {path = ".",extras = ["devbase","test","docs","dist"] [packages] flask-container-scaffold = {path = ".",editable = true} +[celery] +flask-container-scaffold = {file = ".", editable = true, extras = ["celery"]} diff --git a/README.md b/README.md index d702c68..87ce26b 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,51 @@ example: }, }) +### CeleryScaffold + +This class has all of the same support as the above AppScaffold and takes +the same parameters. Each CeleryScaffold instance has a flask_app and celery_app +attribute that can be used in your project. More information about celery can +be found [here](https://docs.celeryq.dev/en/stable/getting-started/introduction.html). +Information on integrating celery with flask can be found in flask's +[documentation](https://flask.palletsprojects.com/en/2.3.x/patterns/celery/). + + +#### Installation + + pip install flask-container-scaffold['celery'] + +or + + pipenv install --categories celery + +#### Basic Usage + + celery_scaffold = CeleryScaffold(name=__name__, config=config) + flask_app = celery_scaffold.flask_app + celery_app = celery_scaffold.celery_app + +#### Basic Configuration + +All configuration is done via a 'CELERY' key in a configuration dictionary. The +'CELERY' element itself is a dictionary of configuration items. More details on the +available configuration items for celery can be found [here](https://docs.celeryq.dev/en/stable/userguide/configuration.html). +Below is a basic example in yaml format that uses a local rabbitmq broker, json serialization, and no result backend. + +``` +--- + +CELERY: + broker: "pyamqp://guest@127.0.0.1//" + result_persistent: False + task_serializer: "json" + accept_content: + - "json" # Ignore other content + result_serializer: "json" + result_expires: "300" + broker_connection_retry_on_startup: 'False' +``` + ### Using the parse_input method This method is used to validate incoming data against a pydantic model. A diff --git a/setup.cfg b/setup.cfg index 4ad70ee..63cb978 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,9 @@ install_requires = toolchest [options.extras_require] +celery = + celery + devbase = tox diff --git a/src/flask_container_scaffold/app_scaffold.py b/src/flask_container_scaffold/app_scaffold.py index 2c6c5b2..ff4571f 100644 --- a/src/flask_container_scaffold/app_scaffold.py +++ b/src/flask_container_scaffold/app_scaffold.py @@ -1,11 +1,7 @@ -import os +from flask_container_scaffold.base_scaffold import BaseScaffold -from flask import Flask -from flask_container_scaffold.app_configurator import AppConfigurator - - -class AppScaffold(object): +class AppScaffold(BaseScaffold): def __init__(self, app=None, name=__name__, config=None, @@ -13,86 +9,10 @@ def __init__(self, app=None, instance_path=None, instance_relative_config=True): """ - This class provides a way to dynamically configure a Flask application. - - :param obj app: An existing Flask application, if passed, otherwise we - will create a new one - :param str name: The name of the application, defaults to __name__. - :param dict config: A dict of configuration details. This can include - standard Flask configuration keys, like 'TESTING', or - 'CUSTOM_SETTINGS' (which can be a string referencing a file with custom - configuration, or a dictionary containing any values your application - may need) to make them available to the application during runtime - :param bool settings_required: Whether your app requires certain - settings be specified in a settings.cfg file - :param str instance_path: Passthrough parameter to flask. An - alternative instance path for the application. By default - the folder 'instance' next to the package or module is - assumed to be the instance path. - :param bool instance_relative_config: Passthrough parameter to flask. - If set to True relative filenames for loading the config - are assumed to be relative to the instance path instead of - the application root. - - """ - # TODO: Consider taking **kwargs here, so we can automatically support - # all params the flask object takes, and just pass them through. Keep - # the ones we already have, as they are needed for the current code to - # work. - Flask.jinja_options = dict(Flask.jinja_options, trim_blocks=True, - lstrip_blocks=True) - self.app = (app or - Flask(name, - instance_relative_config=instance_relative_config, - instance_path=instance_path)) - self.config = config - self.silent = not settings_required - self.relative = instance_relative_config - self._init_app() - - def _init_app(self): - self._load_flask_settings() - self._load_custom_settings() - - def _load_flask_settings(self): - """ - This loads the 'core' settings, ie, anything you could set directly - on a Flask app. These can be specified in the following order, each - overriding the last, if specified: - - via config mapping - - via Flask settings.cfg file - - via environment variable 'FLASK_SETTINGS' - """ - config_not_loaded = True - if self.config is not None: - # load the config if passed in - self.app.config.from_mapping(self.config) - config_not_loaded = False - # load the instance config, if it exists and/or is required - try: - self.app.config.from_pyfile('settings.cfg', silent=self.silent) - config_not_loaded = False - except Exception: - config_not_loaded = True - # Load any additional config specified in the FLASK_SETTINGS file, - # if it exists. We only want to fail in the case where settings are - # required by the app. - if ((config_not_loaded and not self.silent) or - os.environ.get('FLASK_SETTINGS')): - self.app.config.from_envvar('FLASK_SETTINGS') - - def _load_custom_settings(self): - """ - Load any custom configuration for the app from: - - app.config['CUSTOM_SETTINGS'] - - environment variable 'CUSTOM_SETTINGS' + This class provides compatibility with versions of scaffold that + expect an instance with an 'app' attribute. All of the parameters are + the same as BaseScaffold and are passed directly through unmodified. """ - configurator = AppConfigurator(self.app, self.relative) - if self.app.config.get('CUSTOM_SETTINGS') is not None: - # load the config if passed in - custom = self.app.config.get('CUSTOM_SETTINGS') - configurator.parse(custom) - # Next, load from override file, if specified - if os.environ.get('CUSTOM_SETTINGS') is not None: - custom = os.environ.get('CUSTOM_SETTINGS') - configurator.parse(custom) + super().__init__(app, name, config, settings_required, + instance_path, instance_relative_config) + self.app = app or self.flask_app diff --git a/src/flask_container_scaffold/base_scaffold.py b/src/flask_container_scaffold/base_scaffold.py new file mode 100644 index 0000000..208aec7 --- /dev/null +++ b/src/flask_container_scaffold/base_scaffold.py @@ -0,0 +1,102 @@ +import os + +from flask import Flask + +from flask_container_scaffold.app_configurator import AppConfigurator + + +class BaseScaffold(object): + + def __init__(self, app=None, + name=__name__, config=None, + settings_required=False, + instance_path=None, + instance_relative_config=True): + """ + This base class provides a way to dynamically configure a Flask + application. + + :param obj app: An existing Flask application, if passed, otherwise we + will create a new one + :param str name: The name of the application, defaults to __name__. + :param dict config: A dict of configuration details. This can include + standard Flask configuration keys, like 'TESTING', or + 'CUSTOM_SETTINGS' (which can be a string referencing a file with + custom configuration, or a dictionary containing any values your + application may need) to make them available to the application + during runtime + :param bool settings_required: Whether your app requires certain + settings be specified in a settings.cfg file + :param str instance_path: Passthrough parameter to flask. An + alternative instance path for the application. By default + the folder 'instance' next to the package or module is + assumed to be the instance path. + :param bool instance_relative_config: Passthrough parameter to flask. + If set to True relative filenames for loading the config + are assumed to be relative to the instance path instead of + the application root. + + """ + # TODO: Consider taking **kwargs here, so we can automatically support + # all params the flask object takes, and just pass them through. Keep + # the ones we already have, as they are needed for the current code to + # work. + Flask.jinja_options = dict(Flask.jinja_options, trim_blocks=True, + lstrip_blocks=True) + self.flask_app = app or Flask( + name, + instance_relative_config=instance_relative_config, + instance_path=instance_path, + ) + self.config = config + self.silent = not settings_required + self.relative = instance_relative_config + self._init_app() + + def _init_app(self): + self._load_flask_settings() + self._load_custom_settings() + + def _load_flask_settings(self): + """ + This loads the 'core' settings, ie, anything you could set directly + on a Flask app. These can be specified in the following order, each + overriding the last, if specified: + - via config mapping + - via Flask settings.cfg file + - via environment variable 'FLASK_SETTINGS' + """ + config_not_loaded = True + if self.config is not None: + # load the config if passed in + self.flask_app.config.from_mapping(self.config) + config_not_loaded = False + # load the instance config, if it exists and/or is required + try: + self.flask_app.config.from_pyfile('settings.cfg', + silent=self.silent) + config_not_loaded = False + except Exception: + config_not_loaded = True + # Load any additional config specified in the FLASK_SETTINGS file, + # if it exists. We only want to fail in the case where settings are + # required by the app. + if ((config_not_loaded and not self.silent) or + os.environ.get('FLASK_SETTINGS')): + self.flask_app.config.from_envvar('FLASK_SETTINGS') + + def _load_custom_settings(self): + """ + Load any custom configuration for the app from: + - app.config['CUSTOM_SETTINGS'] + - environment variable 'CUSTOM_SETTINGS' + """ + configurator = AppConfigurator(self.flask_app, self.relative) + if self.flask_app.config.get('CUSTOM_SETTINGS') is not None: + # load the config if passed in + custom = self.flask_app.config.get('CUSTOM_SETTINGS') + configurator.parse(custom) + # Next, load from override file, if specified + if os.environ.get('CUSTOM_SETTINGS') is not None: + custom = os.environ.get('CUSTOM_SETTINGS') + configurator.parse(custom) diff --git a/src/flask_container_scaffold/celery_scaffold.py b/src/flask_container_scaffold/celery_scaffold.py new file mode 100644 index 0000000..0a12690 --- /dev/null +++ b/src/flask_container_scaffold/celery_scaffold.py @@ -0,0 +1,30 @@ +from celery import Celery + +from flask_container_scaffold.base_scaffold import BaseScaffold + + +class CeleryScaffold(BaseScaffold): + + def __init__(self, flask_app=None, name=__name__, config=None, + settings_required=False, + instance_path=None, + instance_relative_config=True): + """ + This class provides both a flask 'app' and a celery 'app' that has been + configured via flask. All of the parameters are the same as BaseScaffold. + Any naming changes are noted below. + + :param obj flask_app: An existing Flask application, if passed, + otherwise we will create a new one using BaseScaffold. This is the same + as the app parameter in BaseScaffold. + """ + super().__init__(flask_app, name, config, settings_required, + instance_path, instance_relative_config) + self.flask_app = flask_app or self.flask_app + self.celery_app = Celery(self.flask_app.name) + self.celery_app.config_from_object(self.flask_app.config.get("CELERY")) + self.celery_app.set_default() + # Add the celery app as an extension to the flask app so it can be easily + # accessed if a flask application factory pattern is used. + # see https://flask.palletsprojects.com/en/2.3.x/patterns/celery/ for details. + self.flask_app.extensions["celery"] = self.celery_app diff --git a/test-requirements.txt b/test-requirements.txt index 53e3c74..3f5dc8a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,2 @@ -i https://pypi.python.org/simple --e .[devbase,test] +-e .[devbase,test,celery] diff --git a/tests/unit/test_celery.py b/tests/unit/test_celery.py new file mode 100644 index 0000000..d7652a1 --- /dev/null +++ b/tests/unit/test_celery.py @@ -0,0 +1,65 @@ +import pytest + +from celery import Celery +from flask import Flask + +from flask_container_scaffold.celery_scaffold import CeleryScaffold + + +def test_celery_flask_empty_config(): + """ + GIVEN an instance of CeleryScaffold with an empty config + WHEN we try to create the app + THEN we get a celery app and a flask app + """ + scaffold = CeleryScaffold() + assert scaffold.flask_app is not None + assert isinstance(scaffold.flask_app, Flask) + assert scaffold.celery_app is not None + assert isinstance(scaffold.celery_app, Celery) + + +def test_flask_extension(): + """ + Given an instance of CeleryScaffold + WHEN the apps are created + THEN the flask extension has a celery element + AND the celery app matches the flask extension + """ + scaffold = CeleryScaffold() + assert scaffold.flask_app is not None + assert scaffold.celery_app is not None + assert scaffold.flask_app.extensions.get("celery") is not None + assert scaffold.celery_app == scaffold.flask_app.extensions["celery"] + + +def test_celery_broker_set(): + """ + GIVEN an instance of CeleryScaffold + AND a config with a broker url + WHEN we create the app + THEN we get a celery app with a broker url matching the config + """ + config = {'CELERY': {'broker': 'pyamqp://'}} + scaffold = CeleryScaffold(config=config) + app = scaffold.celery_app + assert app is not None + assert isinstance(app, Celery) + assert config['CELERY']['broker'] == app.conf.find_value_for_key('broker') + + +def test_celery_bad_config(): + """ + GIVEN an instance of CeleryScaffold + AND a config with a bad config item + WHEN we create the app + THEN we get a celery app + AND the config doesn't have the bad item + """ + config = {'CELERY': {'bad_config_item': 'my_bad_config'}} + scaffold = CeleryScaffold(config=config) + app = scaffold.celery_app + assert app is not None + assert isinstance(app, Celery) + with pytest.raises(KeyError): + app.conf.find_value_for_key('bad_config_item')