Enable DB replica

.circleci/config.yml

@@ -193,7 +193,8 @@ jobs:
           name: Run pytest
           command: |
             source $(pyenv root)/versions/3.10.4/envs/env/bin/activate
-            ENVIRONMENT=test pytest --cov-report xml:test-results/coverage.xml --cov=boxtribute_server
+            # overwrite environment variable because it must not depend on the CircleCI context
+            ENVIRONMENT=development MYSQL_PORT=3306 pytest --cov-report xml:test-results/coverage.xml --cov=boxtribute_server
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -242,12 +243,11 @@ jobs:
               AUTH0_AUDIENCE=${AUTH0_AUDIENCE}
               AUTH0_JWKS_KID=${AUTH0_JWKS_KID}
               AUTH0_JWKS_N=${AUTH0_JWKS_N}
-              MYSQL_HOST=127.0.0.1
               MYSQL_USER=${DB_NAME}
               MYSQL_PASSWORD=${DB_PASS}
               MYSQL_DB=${DB_NAME}
-              MYSQL_PORT=3306
               MYSQL_SOCKET=${DB_SOCKET}
+              MYSQL_REPLICA_SOCKET=${DB_REPLICA_SOCKET}
               SENTRY_DSN=${SENTRY_BE_DSN}
               SENTRY_ENVIRONMENT=${ENVIRONMENT}
               SENTRY_RELEASE=${CIRCLE_SHA1}
@@ -314,12 +314,11 @@ jobs:
               AUTH0_CLIENT_SECRET=${QUERY_API_AUTH0_CLIENT_SECRET}
               AUTH0_JWKS_KID=${AUTH0_JWKS_KID}
               AUTH0_JWKS_N=${AUTH0_JWKS_N}
-              MYSQL_HOST=127.0.0.1
               MYSQL_USER=${DB_NAME}
               MYSQL_PASSWORD=${DB_PASS}
               MYSQL_DB=${DB_NAME}
-              MYSQL_PORT=3306
               MYSQL_SOCKET=${DB_SOCKET}
+              MYSQL_REPLICA_SOCKET=${DB_REPLICA_SOCKET}
               SENTRY_DSN=${SENTRY_BE_DSN}
               SENTRY_ENVIRONMENT=${ENVIRONMENT}
               SENTRY_RELEASE=${CIRCLE_SHA1}

back/README.md

@@ -206,9 +206,9 @@ Most tests require a running MySQL server. Before executing tests for the first
 
 Run the test suite on your machine by executing
 
-    MYSQL_PORT=32000 pytest
+    pytest
 
-If you persistently want these variables to be set for your environment, export them via the `.envrc` file.
+Add `-x` to stop at the first failure, and `-v` or `-vv` for increased verbosity.
 
 You can also run the tests via `docker-compose`:
 

back/boxtribute_server/app.py

@@ -14,14 +14,24 @@ def create_app():
     return Flask(__name__)
 
 
-def configure_app(app, *blueprints, database_interface=None, **mysql_kwargs):
+def configure_app(
+    app, *blueprints, database_interface=None, replica_socket=None, **mysql_kwargs
+):
     """Register blueprints. Configure the app's database interface. `mysql_kwargs` are
     forwarded.
     """
     for blueprint in blueprints:
         app.register_blueprint(blueprint)
 
     app.config["DATABASE"] = database_interface or create_db_interface(**mysql_kwargs)
+
+    if replica_socket or mysql_kwargs:
+        # In deployed environment: replica_socket is set
+        # In integration tests: connect to same host/port as primary database
+        # In endpoint tests, no replica connection is used
+        mysql_kwargs["unix_socket"] = replica_socket
+        app.config["DATABASE_REPLICA"] = create_db_interface(**mysql_kwargs)
+
     db.init_app(app)
 
 
@@ -53,7 +63,10 @@ def before_sentry_send(event, hint):
             return
         return event
 
-    # dsn/environment/release: reading SENTRY_* environment variables set in CircleCI
+    # The SDK requires the parameters dns, and optionally, environment and release for
+    # initialization. In the deployed GAE environments they are read from the
+    # environment variables `SENTRY_*`. Since in local or CI testing environments these
+    # variables don't exist, the SDK is not effective which is desired.
     sentry_sdk.init(
         integrations=[FlaskIntegration(), AriadneIntegration()],
         traces_sample_rate=float(os.getenv("SENTRY_TRACES_SAMPLE_RATE", 0.0)),
@@ -65,11 +78,15 @@ def before_sentry_send(event, hint):
     configure_app(
         app,
         *blueprints,
-        host=os.environ["MYSQL_HOST"],
-        port=int(os.environ["MYSQL_PORT"]),
+        # always used
         user=os.environ["MYSQL_USER"],
         password=os.environ["MYSQL_PASSWORD"],
         database=os.environ["MYSQL_DB"],
+        # used for connecting to development / CI testing DB
+        host=os.getenv("MYSQL_HOST"),
+        port=int(os.getenv("MYSQL_PORT", 0)),
+        # used for connecting to Google Cloud from GAE
         unix_socket=os.getenv("MYSQL_SOCKET"),
+        replica_socket=os.getenv("MYSQL_REPLICA_SOCKET"),
     )
     return app

back/boxtribute_server/db.py

@@ -1,15 +1,61 @@
 from peewee import MySQLDatabase
 from playhouse.flask_utils import FlaskDB  # type: ignore
 
-db = FlaskDB()
+
+class DatabaseManager(FlaskDB):
+    """Custom class to glue Flask and Peewee together.
+    If configured accordingly, connect to a database replica, and use it in all SELECT
+    SQL queries.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.replica = None
+
+    def init_app(self, app):
+        self.replica = app.config.get("DATABASE_REPLICA")  # expecting peewee.Database
+        super().init_app(app)
+
+    def connect_db(self):
+        super().connect_db()
+        if self.replica:
+            self.replica.connect()
+
+    def close_db(self, exc):
+        super().close_db(exc)
+        if self.replica and not self.replica.is_closed():
+            self.replica.close()
+
+    def get_model_class(self):
+        """Whenever a database model (representing a table) is defined, it must derive
+        from `db.Model` which calls this method.
+        """
+        if self.database is None:
+            raise RuntimeError("Database must be initialized.")
+
+        class BaseModel(self.base_model_class):
+            @classmethod
+            def select(cls, *args, **kwargs):
+                if self.replica:
+                    with cls.bind_ctx(self.replica):
+                        return super().select(*args, **kwargs)
+                return super().select(*args, **kwargs)
+
+            class Meta:
+                database = self.database
+
+        return BaseModel
+
+
+db = DatabaseManager()
 
 
 def create_db_interface(**mysql_kwargs):
     """Create MySQL database interface using given connection parameters. `mysql_kwargs`
     are validated to not be None and forwarded to `pymysql.connect`.
     Configure primary keys to be unsigned integer.
     """
-    for field in ["host", "port", "user", "password", "database"]:
+    for field in ["user", "password", "database"]:
         if mysql_kwargs.get(field) is None:
             raise ValueError(
                 f"Field '{field}' for database configuration must not be None"

back/setup.cfg

@@ -24,7 +24,6 @@ exclude_lines =
     pragma: no cover
     if __name__ == .__main__.:
     except Exception:
-    def main()
 omit =
     */*main.py
 

back/test/conftest.py

@@ -14,7 +14,7 @@
 
 import pymysql
 import pytest
-from boxtribute_server.app import configure_app, create_app
+from boxtribute_server.app import configure_app, create_app, main
 from boxtribute_server.db import create_db_interface, db
 from boxtribute_server.routes import api_bp, app_bp
 
@@ -28,7 +28,7 @@
     # - db:3306         when testing in container on local machine
     # - 127.0.0.1:3306  when testing in CircleCI
     host=os.getenv("MYSQL_HOST", "127.0.0.1"),
-    port=int(os.getenv("MYSQL_PORT", 3306)),
+    port=int(os.getenv("MYSQL_PORT", 32000)),
     user="root",
     password="dropapp_root",
 )
@@ -110,23 +110,24 @@ def client(mysql_testing_database):
 
 
 @pytest.fixture
-def dropapp_dev_client():
+def dropapp_dev_client(monkeypatch):
     """Function fixture for any tests that include read-only operations on the
     `dropapp_dev` database. Use for testing the integration of the webapp (and the
     underlying ORM) with the format of the dropapp production database.
     The fixture creates a web app (exposing both the query and the full API), configured
     to connect to the `dropapp_dev` MySQL database, and returns a client that simulates
     sending requests to the app.
     """
-    app = create_app()
+    monkeypatch.setenv("MYSQL_HOST", MYSQL_CONNECTION_PARAMETERS["host"])
+    monkeypatch.setenv("MYSQL_PORT", str(MYSQL_CONNECTION_PARAMETERS["port"]))
+    monkeypatch.setenv("MYSQL_USER", MYSQL_CONNECTION_PARAMETERS["user"])
+    monkeypatch.setenv("MYSQL_PASSWORD", MYSQL_CONNECTION_PARAMETERS["password"])
+    monkeypatch.setenv("MYSQL_DB", "dropapp_dev")
+    monkeypatch.setenv("MYSQL_SOCKET", "")
+    monkeypatch.setenv("MYSQL_REPLICA_SOCKET", "")
+
+    app = main(api_bp, app_bp)
     app.testing = True
-    configure_app(
-        app,
-        api_bp,
-        app_bp,
-        **MYSQL_CONNECTION_PARAMETERS,
-        database="dropapp_dev",
-    )
 
     with db.database.bind_ctx(MODELS):
         db.database.create_tables(MODELS)

back/test/model_tests/conftest.py

@@ -10,9 +10,12 @@
 def setup_db_before_test(mysql_testing_database, mocker):
     """Automatically create all tables before each test and populate them, and drop all
     tables at tear-down.
-    Patch the wrapped database of the FlaskDB because it is uninitialized otherwise.
+    Patch the wrapped database of the DatabaseManager because it is uninitialized
+    otherwise, and the replica database (might be set on the global DatabaseManager
+    when having run integration tests before.
     """
     mocker.patch.object(db, "database", mysql_testing_database)
+    mocker.patch.object(db, "replica", None)
     with mysql_testing_database.bind_ctx(MODELS):
         mysql_testing_database.drop_tables(MODELS)
         mysql_testing_database.create_tables(MODELS)

back/test/model_tests/test_utils.py

@@ -2,8 +2,15 @@
 from boxtribute_server.business_logic.box_transfer.shipment.fields import (
     first_letters_of_base_name,
 )
+from boxtribute_server.db import DatabaseManager
 
 
 @pytest.mark.parametrize("base_id,result", [[1, "TH"], [2, "AS"], [3, "WÜ"]])
 def test_first_letters_of_base_name(base_id, result):
     assert first_letters_of_base_name(base_id) == result
+
+
+def test_unitialized_database_manager():
+    manager = DatabaseManager()
+    with pytest.raises(RuntimeError):
+        manager.get_model_class()

example.envrc

@@ -1,4 +1,3 @@
 #!/bin/bash
 
 layout virtualenv .venv
-export MYSQL_PORT=32000