Backend Localization (#218)

* Initial fluent support for backend localization:
 - Adds starlette_context so we can safely have per-request data
 - Adds a middleware to read accept-language and load fluent
 - Adds a `l10n` helper function to localize a given key
 - Adjusts the health check to return a distinct string between locales (we use this for tests...)
 - A couple of string -> fluent conversions

* Localize emails, add tests for those emails, and fix test pathing issues

* Install the package instead of manually installing requirements.

* Remove the `if not subscriber` from a load of routes, because we do that in the dependency. Also re-org the exception to a separate file.

* Swap some commonly used exceptions for a validation exception class, and l10n-ify api

* Refactor exception names and definition flow. And replace a few more in schedule.py

* Add a few more exceptions and finish off schedule.py

* Update the zoom.py exceptions

* Update the health check strings

* Remove `DATABASE_SECRETS` which is used for AWS, and add a default database url for docker.

* 🔨 fix legacy env var

* Revert committing frontend testing stash changes on this pr.

* 🌐 add German backend translation

---------

Co-authored-by: Andreas Müller <[email protected]>

.github/workflows/backend-tests.yml

@@ -25,8 +25,7 @@ jobs:
       run: |
         cd ./backend
         python -m pip install --upgrade pip
-        python -m pip install -r requirements.txt
-        python -m pip install -r requirements-test.txt
+        python -m pip install .'[test]'
     - name: Test with pytest
       run: |
-        cd ./backend/test && python -m pytest
+        cd ./backend && python -m pytest

.gitignore

@@ -1,12 +1,13 @@
-.vscode
-__pycache__
-.pytest_cache
-dist
-*.db
-*.ini
-*.log
-*credentials.json
-*.pickle
-.env
-venv
-.idea
+.vscode
+__pycache__
+.pytest_cache
+dist
+*.db
+*.ini
+*.log
+*credentials.json
+*.pickle
+.env
+venv
+.idea
+*.egg-info

README.md

@@ -43,10 +43,10 @@ Run application for development with hot reloading backend and frontend:
 
     ```bash
     cd appointment
-    pip install -r backend/requirements.txt
+    pip install .
     touch backend/src/appointment.db # when using sqlite
     cp backend/.env.example backend/.env # add your own configuration here
-    uvicorn --factory backend.src.appointment.main:server --host 0.0.0.0 --port 5000
+    uvicorn --factory appointment.main:server --host 0.0.0.0 --port 5000
     ```
 
     You can now access the backend at [localhost:5000](http://localhost:5000).
@@ -74,16 +74,16 @@ Run application for development with hot reloading backend and frontend:
 
 ## Testing
 
-To run tests, setup the application manually (you don't need the mysql deps), and then install requirements-test.txt
+To run tests, simply install the package in editing mode:
 
 ```bash
-pip install -r requirements-test.txt
+cd backend && pip install -e .
 ```
 
 After this you can run tests with:
 
 ```bash
-cd backend/test && python -m pytest
+cd backend && python -m pytest
 ```
 
 ## Contributing
@@ -116,3 +116,7 @@ Commands (from /frontend)
 yarn run lint
 yarn run lint --fix
 ```
+
+### Localization
+
+This project uses [Fluent](https://projectfluent.org/) for localization. Files are located in their respective `l10n/<locale>/*.ftl`.

backend/.env.example

@@ -11,8 +11,7 @@ FRONTEND_URL=http://localhost:8080
 SHORT_BASE_URL=
 
 # -- DATABASE --
-DATABASE_URL=
-DATABASE_SECRETS=
+DATABASE_URL="mysql+mysqldb://tba:tba@mysql:3306/appointment"
 # Secret phrase for database encryption (e.g. create it by running `openssl rand -hex 32`)
 DB_SECRET=
 

backend/Dockerfile

@@ -1,23 +1,28 @@
-FROM python:3.11-buster
-
-RUN mkdir app
-WORKDIR /app
-
-ENV PATH="${PATH}:/root/.local/bin"
-ENV PYTHONPATH=.
-
-RUN mkdir scripts
-
-COPY requirements.txt .
-COPY pyproject.toml .
-COPY alembic.ini.example alembic.ini
-COPY scripts/dev-entry.sh scripts/dev-entry.sh
-
-# Dev only
-COPY .env .
-
-RUN pip install --upgrade pip
-RUN pip install .
-
-EXPOSE 5000
-CMD ["/bin/sh", "./scripts/dev-entry.sh"]
+FROM python:3.11-buster
+
+RUN mkdir app
+WORKDIR /app
+
+ENV PATH="${PATH}:/root/.local/bin"
+ENV PYTHONPATH=.
+
+RUN mkdir scripts
+
+COPY requirements.txt .
+COPY pyproject.toml .
+COPY alembic.ini.example alembic.ini
+COPY scripts/dev-entry.sh scripts/dev-entry.sh
+
+# Dev only
+COPY .env .
+
+RUN pip install --upgrade pip
+RUN pip install .'[deploy]'
+
+# Add this hack to line it up with our dev environment.
+# I'll buy whoever fixes this a coffee.
+RUN mkdir src
+RUN ln -s /app/appointment src/appointment
+
+EXPOSE 5000
+CMD ["/bin/sh", "./scripts/dev-entry.sh"]

backend/alembic.ini.example

@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = src/appointment/migrations
+script_location = appointment/migrations
 
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time
@@ -12,7 +12,7 @@ file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(re
 
 # sys.path path, will be prepended to sys.path if present.
 # defaults to the current working directory.
-prepend_sys_path = src/appointment
+prepend_sys_path = appointment
 
 # timezone to use when rendering the date within the migration file
 # as well as the filename.

backend/deploy.dockerfile

@@ -17,7 +17,7 @@ COPY scripts/entry.sh scripts/entry.sh
 COPY src .
 
 RUN pip install --upgrade pip
-RUN pip install .
+RUN pip install .'[deploy]'
 
 # install removes the src file and installs the application as /app/appointment
 # that's fine, but uhh let's add this hack to line it up with our dev environment.
@@ -26,4 +26,4 @@ RUN mkdir src
 RUN ln -s /app/appointment src/appointment
 
 EXPOSE 5000
-CMD ["/bin/sh", "./scripts/entry.sh"]
+CMD ["/bin/sh", "./scripts/entry.sh"]

backend/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
-name = "appointment_backend"
+name = "appointment"
 version = "0.2.0"
 description = "Backend component to Thunderbird Appointment"
 requires-python = ">3.11"
 dynamic = ["dependencies"]
 
 [project.scripts]
-run-command = "src.appointment.main:cli"
+run-command = "appointment.main:cli"
 
 [project.urls]
 homepage = "https://appointment.day"
@@ -17,6 +17,16 @@ cli = [
     "ruff",
     "black"
 ]
+db = [
+    "mysqlclient==2.1.1",
+    "mysql-connector-python==8.0.32",
+]
+test = [
+    "Faker==20.1.0",
+    "httpx==0.25.1",
+    "pytest==7.4.3",
+]
+deploy = ['appointment[cli]', 'appointment[db]']
 
 [tool.setuptools.dynamic]
 dependencies = { file = ["requirements.txt"] }
@@ -70,3 +80,6 @@ max-complexity = 10
 
 [tool.black]
 line-length = 120
+
+[tool.pytest.ini_options]
+pythonpath = "test"

backend/requirements.txt

@@ -1,23 +1,25 @@
 alembic==1.9.3
 argon2-cffi==23.1.0
 argon2-cffi-bindings==21.2.0
+Babel==2.14.0
 caldav==1.0.1
 cryptography==39.0.1
 fastapi-auth0==0.3.2
 fastapi==0.104.1
+fluent.runtime==0.4.0
+fluent.syntax==0.19.0
 google-api-python-client==2.85.0
 google-auth-httplib2==0.1.0
 google-auth-oauthlib==1.0.0
 jinja2==3.1.2
 icalendar==5.0.4
 itsdangerous==2.1.2
-mysqlclient==2.1.1
-mysql-connector-python==8.0.32
 python-dotenv==1.0.0
 python-jose==3.3.0
 python-multipart==0.0.6
 pydantic==2.5.2
 sentry-sdk==1.26.0
+starlette-context==0.3.6
 sqlalchemy-utils==0.39.0
 sqlalchemy==1.4.40
 uvicorn==0.20.0

backend/scripts/dev-entry.sh

@@ -6,4 +6,5 @@ run-command update-db
 python -u -m smtpd -n -c DebuggingServer localhost:8050 &
 
 # Start up real webserver
-uvicorn --factory src.appointment.main:server --reload --host 0.0.0.0 --port 5173
+uvicorn --factory appointment.main:server --reload --host 0.0.0.0 --port 5173
+

backend/scripts/entry.sh

@@ -2,4 +2,4 @@
 
 run-command update-db
 
-uvicorn --factory src.appointment.main:server --host 0.0.0.0 --port 5000
+uvicorn --factory appointment.main:server --host 0.0.0.0 --port 5000

backend/src/appointment/controller/calendar.py

@@ -13,7 +13,7 @@
 from ..database import schemas
 from ..database.models import CalendarProvider
 from ..controller.mailer import Attachment, InvitationMail
-
+from ..l10n import l10n
 
 DATEFMT = "%Y-%m-%d"
 
@@ -118,10 +118,10 @@ def create_event(
 
         # Place url and phone in desc if available:
         if event.location.url:
-            description.append(f"Join online at: {event.location.url}")
+            description.append(l10n('join-online', {'url': event.location.url}))
 
         if event.location.phone:
-            description.append(f"Join by phone: {event.location.phone}")
+            description.append(l10n('join-phone', {'phone': event.location.phone}))
 
         body = {
             "summary": event.title,

backend/src/appointment/controller/data.py

@@ -1,11 +1,12 @@
 import csv
+import datetime
 from io import StringIO, BytesIO
 from zipfile import ZipFile
 
 from ..database import repo
-from ..database.models import Subscriber
-from ..download_readme import get_download_readme
+from ..database.schemas import Subscriber
 from ..exceptions.account_api import AccountDeletionPartialFail, AccountDeletionSubscriberFail
+from ..l10n import l10n
 
 
 def model_to_csv_buffer(models):
@@ -70,7 +71,7 @@ def download(db, subscriber: Subscriber):
         data_zip.writestr("external_connection.csv", external_connections_buffer.getvalue())
         data_zip.writestr("schedules.csv", schedules_buffer.getvalue())
         data_zip.writestr("availability.csv", availability_buffer)
-        data_zip.writestr("readme.txt", get_download_readme())
+        data_zip.writestr("readme.txt", l10n('account-data-readme', {'download_time': datetime.datetime.now(datetime.UTC)}))
 
     # Return our zip buffer
     return zip_buffer