Merge pull request #28 from lsst-sqre/tickets/DM-42875

DM-42875: Adopt faststream for Kafka producer

.github/workflows/ci.yaml

@@ -16,53 +16,62 @@ name: CI
       - "u/**"
     tags:
       - "*"
+  release:
+    types: [published]
 
 jobs:
   lint:
 
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Run pre-commit
         uses: pre-commit/[email protected]
 
   test:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
 
     strategy:
       matrix:
         python:
-          - "3.11"
+          - "3.12"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ matrix.python }}
 
       - name: Install nox
         run: |
           pip install --upgrade pip
           pip install --upgrade nox
 
+      - name: Start Kafka
+        run: docker compose -f kafka-compose.yaml up -d
+
       - name: Run nox
         run: "nox -s typing test"
 
+      - name: Stop Kafka
+        run: docker compose -f kafka-compose.yaml down
+
   docs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           # Ensure the documentation gets the right version.
           fetch-depth: 0
@@ -74,7 +83,7 @@ jobs:
         run: sudo apt install -y graphviz
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
@@ -117,7 +126,7 @@ jobs:
       || startsWith(github.head_ref, 'tickets/')
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # full history for setuptools_scm
 
@@ -129,20 +138,39 @@ jobs:
 
       - run: echo Pushed ghcr.io/${{ github.repository }}/${{ steps.build.outputs.tag }}
 
-  pypi:
+  test-package:
+    name: Test PyPI package build
+    runs-on: ubuntu-latest
 
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # full history for setuptools_scm
+
+      - name: Build and publish
+        uses: lsst-sqre/build-and-publish-to-pypi@v2
+        with:
+          python-version: "3.12"
+          upload: "false"
+          working-directory: "client"
+
+  pypi-publish:
+    name: Upload release to PyPI
     runs-on: ubuntu-latest
-    needs: [lint, test, docs]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/rubin-squarebot
+    permissions:
+      id-token: write
+    if: github.event_name == 'release' && github.event.action == 'published'
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0 # full history for setuptools_scm
 
       - name: Build and publish
-        uses: lsst-sqre/build-and-publish-to-pypi@v1
+        uses: lsst-sqre/build-and-publish-to-pypi@v2
         with:
-          pypi-token: ${{ secrets.PYPI_SQRE_ADMIN }}
-          python-version: "3.11"
-          upload: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
-          working-directory: "./client"
+          python-version: "3.12"
+          working-directory: "client"

.pre-commit-config.yaml

@@ -1,23 +1,14 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: check-yaml
       - id: check-toml
 
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.2.1
     hooks:
-      - id: isort
-        additional_dependencies: [toml]
-
-  - repo: https://github.com/psf/black
-    rev: 23.3.0
-    hooks:
-      - id: black
-
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format

CHANGELOG.md

@@ -2,6 +2,47 @@
 
 <!-- scriv-insert-here -->
 
+<a id='changelog-0.8.0'></a>
+## 0.8.0 (2024-02-28)
+
+### Backwards-incompatible changes
+
+- Squarebot now publishes messages in JSON, rather than Avro. Consumers should use the Pydantic models published in the `rubin-squarebot` PyPI package to deserialize messages.
+
+- The `rubin-squarebot` PyPI package now uses the `rubin.squarebot` Python namespace (previously `rubinobs.square.squarebot`). Consumers should use the `rubin.squarebot.models.kafka.SquarebotSlackMessageValue` Pydantic model to deserialize Squarebot messages for Slack channel message traffic (or `rubin.squarebot.models.kafka.SquarebotSlackMessageValue` for the app mention topic).
+
+- The codebase now uses Pydantic 2.
+
+### New features
+
+- Squarebot now uses [FastStream](https://faststream.airt.ai/) to publish messages. This approach drops the Confluent Schema Registry integration because messages are now published in JSON from Pydantic models, and those Pydantic models are versioned and published through the `rubin-squarebot` PyPI package.
+
+- New fields in the `SquarebotSlackMessageValue` Pydantic model:
+
+  - `thread_ts`, useful for identify a threaded message and replying to its thread.
+  - `bot_id`, the ID of the bot that sent the message (if applicable). This is useful for identifying bot messages and ignoring them in some cases.
+
+### Bug fixes
+
+- Fix type annotations related in `channel_type` in `rubin.squarebot.models.kafka.SquarebotSlackMessageValue`. We now assure that the channel type is not Null here. However the `channel_type` field is now set to null/None in the `SquarebotSlackAppMentionValue` model.
+
+- Fixed the `nox -s init` command so that it will install into the current Python environment (previously it still installing into the environment managed by `nox`).
+
+### Other changes
+
+- A Redoc-based OpenAPI documentation page for the REST API is now included in the Sphinx documentation.
+
+- Switched to [nox](https://nox.thea.codes/en/stable/) for running tests and repository tasks. Nox now replaces the two tox configurations for the client and server. Nox also replaces the `Makefile` for repository tasks:
+  - `nox -s venv-init` initializes a Python venv virtual environment and installs the application into it. Alternatively, `nox -s init` can be used to initialize the application in the current Python environment (like `make init`).
+  - `nox -s update-deps` updates the pinned dependencies as well as the pre-commit hooks (replacing `make update-deps`).
+- New nox integration with scriv for change log management: `nox -s scriv-create` creates a change log fragment and `nox -s scriv-collect X.Y.Z` collects change log fragments into CHANGELOG.md for a release.
+
+- Adopt the lsst-sqre/build-and-push-to-ghcr@v1 action.
+
+- Tests require a local Kafka broker. The easiest way to run a Kafka broker is with Docker Compose and the `kafka-compose.yaml` configuration file in this repository. The Docker compose set up also deploys Kafdrop so you can view the messages produced during testing.
+
+- The codebase is now linted and formatted with Ruff.
+
 <a id='changelog-0.7.0'></a>
 
 ## 0.7.0 (2023-05-19)

Dockerfile

@@ -14,7 +14,7 @@
 #   - Runs a non-root user.
 #   - Sets up the entrypoint and port.
 
-FROM python:3.11.4-slim-bullseye as base-image
+FROM python:3.12.2-slim-bullseye as base-image
 
 # Update system packages
 COPY scripts/install-base-packages.sh .

README.md

@@ -43,4 +43,47 @@ flowchart LR
 Slack integration is implemented at the time.
 We plan to add support for other event sources, such as GitHub, in the future.
 
-Squarebot is built on top of [FastAPI](https://fastapi.tiangolo.com/), a modern Python web framework, with Rubin/SQuaRE libraries [Safir](https://safir.lsst.io) (FastAPI application libraries) and [Kafkit](https://kafkit.lsst.io) (Kafka integration).
+Squarebot is built on top of [FastAPI](https://fastapi.tiangolo.com/), a modern Python web framework, with the Rubin/SQuaRE [Safir](https://safir.lsst.io) library.
+Squarebot uses [FastStream](https://faststream.airt.ai/latest/) to publish messages to Kafka with the [aiokafka](https://aiokafka.readthedocs.io/en/stable/) library.
+
+## Development set up
+
+Squarebot uses Python 3.12 or later.
+
+You'll need [nox](https://nox.thea.codes/en/stable/) to run the development environment:
+
+```bash
+pip install nox
+```
+
+Then you can set up a virtual environment with the development dependencies:
+
+```bash
+nox -s venv-init`
+```
+
+(If you already have a virtual environment, you can activate it and run `nox -s init` instead).
+
+The tests require a local Kafka broker.
+If you have Docker, you can deploy a Kafka broker with Docker Compose:
+
+```bash
+docker-compose -f kafka-compose.yaml up
+```
+
+To run all linters, type checking, and tests, run:
+
+```bash
+nox
+```
+
+While developing and running tests, you can inspect messages produced to Kafka with Kafdrop at [http://localhost:9000](http://localhost:9000).
+
+You can run individual sessions with `nox -s <session-name>`.
+See `nox -l` for a list of available sessions.
+
+To tear down the Kafka broker, run:
+
+```bash
+docker-compose -f kafka-compose.yaml down
+```

client/pyproject.toml

@@ -18,7 +18,7 @@ classifiers = [
     "Typing :: Typed",
 ]
 requires-python = ">=3.11"
-dependencies = ["pydantic", "safir"]
+dependencies = ["pydantic>=2", "safir"]
 dynamic = ["version"]
 
 [project.optional-dependencies]
@@ -44,10 +44,10 @@ root = "../"
 [tool.coverage.run]
 parallel = true
 branch = true
-source = ["rubinobs.square.squarebot"]
+source = ["rubin.squarebot"]
 
 [tool.coverage.paths]
-source = ["src", ".tox/*/site-packages"]
+source = ["src", "../.nox/test/lib/*/site-packages"]
 
 [tool.coverage.report]
 show_missing = true
@@ -63,48 +63,8 @@ exclude_lines = [
     "if TYPE_CHECKING:",
 ]
 
-[tool.black]
-line-length = 79
-exclude = '''
-/(
-    \.eggs
-  | \.git
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | _build
-  | build
-  | dist
-)/
-'''
-# Use single-quoted strings so TOML treats the string like a Python r-string
-#  Multi-line strings are implicitly treated by black as regular expressions
-
-[tool.pydocstyle]
-# Reference: http://www.pydocstyle.org/en/stable/error_codes.html
-convention = "numpy"
-add_select = [
-    "D212", # Multi-line docstring summary should start at the first line
-]
-add-ignore = [
-    "D105", # Missing docstring in magic method
-    "D102", # Missing docstring in public method (needed for docstring inheritance)
-    "D100", # Missing docstring in public module
-    # Below are required to allow multi-line summaries.
-    "D200", # One-line docstring should fit on one line with quotes
-    "D205", # 1 blank line required between summary line and description
-    "D400", # First line should end with a period
-    # Properties shouldn't be written in imperative mode. This will be fixed
-    # post 6.1.1, see https://github.com/PyCQA/pydocstyle/pull/546
-    "D401",
-]
-
-[tool.isort]
-profile = "black"
-line_length = 79
-# This tox config might be used for the server isort with pre-commit; so add safir demo as a firstparty
-known_first_party = ["rubinobs.square.squarebot", "squarebot", "tests"]
-skip = ["docs/conf.py"]
+[tool.ruff]
+extend = "../pyproject.toml"
 
 [tool.pytest.ini_options]
 asyncio_mode = "strict"