db: configurable connection pool

Allows to configure parameters of DB connection pool for better
optimistic disconnect handling. Closes #58. Closes #133.

CHANGES.rst

@@ -7,6 +7,7 @@ Version 0.8.0 (UNRELEASED)
 - Adds new disk usage retrieval methods using canonical (bytes) and human-readable (KiB) units. (``User``, ``Workflow``)
 - Adds new properties ``started_at`` and ``finished_at`` to the ``Job`` model, updated on status change.
 - Adds ``get_priority`` workflow method, that combines both complexity and concurrency, to pass to the scheduler.
+- Adds a possibility to configure database connection pool parameters via environment variables.
 - Changes disk quota calculation functions to allow passing raw bytes to increase the used quota.
 - Changes to PostgreSQL 12.8.
 - Removes support for Python 2.

reana_db/config.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2018 CERN.
+# Copyright (C) 2018, 2021 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -45,6 +45,22 @@
 )
 """SQLAlchemy database location."""
 
+SQLALCHEMY_MAX_OVERFLOW = int(float(os.getenv("SQLALCHEMY_MAX_OVERFLOW", 2)))
+"""How many new connections can temporarily exceed the pool size?"""
+
+SQLALCHEMY_POOL_PRE_PING = (
+    os.getenv("SQLALCHEMY_POOL_PRE_PING", "False").lower() == "true"
+)
+"""Do we always pre-ping for pessimistic connection handling?"""
+
+SQLALCHEMY_POOL_RECYCLE = int(float(os.getenv("SQLALCHEMY_POOL_RECYCLE", 3600)))
+"""How many seconds a connection can persist?"""
+
+SQLALCHEMY_POOL_SIZE = int(float(os.getenv("SQLALCHEMY_POOL_SIZE", 5)))
+"""How many permanent connections to the database to keep?"""
+
+SQLALCHEMY_POOL_TIMEOUT = int(float(os.getenv("SQLALCHEMY_POOL_TIMEOUT", 30)))
+"""How many seconds to wait when retrieving a new connection from the pool?"""
 
 DEFAULT_QUOTA_RESOURCES = {
     "cpu": "processing time",

reana_db/database.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of REANA.
-# Copyright (C) 2018 CERN.
+# Copyright (C) 2018, 2019, 2020, 2021 CERN.
 #
 # REANA is free software; you can redistribute it and/or modify it
 # under the terms of the MIT License; see LICENSE file for more details.
@@ -15,11 +15,25 @@
 from sqlalchemy.schema import CreateSchema
 from sqlalchemy_utils import create_database, database_exists
 
-from .config import SQLALCHEMY_DATABASE_URI
+from .config import (
+    SQLALCHEMY_DATABASE_URI,
+    SQLALCHEMY_MAX_OVERFLOW,
+    SQLALCHEMY_POOL_PRE_PING,
+    SQLALCHEMY_POOL_RECYCLE,
+    SQLALCHEMY_POOL_SIZE,
+    SQLALCHEMY_POOL_TIMEOUT,
+)
 
 from reana_db.models import Base  # isort:skip  # noqa
 
-engine = create_engine(SQLALCHEMY_DATABASE_URI)
+engine = create_engine(
+    SQLALCHEMY_DATABASE_URI,
+    max_overflow=SQLALCHEMY_MAX_OVERFLOW,
+    pool_pre_ping=SQLALCHEMY_POOL_PRE_PING,
+    pool_recycle=SQLALCHEMY_POOL_RECYCLE,
+    pool_size=SQLALCHEMY_POOL_SIZE,
+    pool_timeout=SQLALCHEMY_POOL_TIMEOUT,
+)
 Session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine))
 Base.query = Session.query_property()