Merge pull request #109 from lsst/tickets/DM-46130

DM-46130: Add tools for comparing two schemas or a schema and a database

.github/workflows/build.yaml

@@ -51,10 +51,10 @@ jobs:
         run: uv pip install --system pytest pytest-xdist pytest-cov
 
       - name: List installed packages
-        run: uv pip list -v
+        run: uv pip list
 
       - name: Build and install
-        run: uv pip install --system -v --no-deps -e .
+        run: uv pip install --system --no-deps -e .
 
       - name: Run tests
         run: pytest -r a -v -n 3 --cov=tests --cov=felis --cov-report=xml --cov-report=term --cov-branch
@@ -91,6 +91,3 @@ jobs:
 
       - name: Upload
         uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_UPLOADS }}

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         # supported by your project here, or alternatively use
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
-        language_version: python3.11
+        language_version: python3.12
   - repo: https://github.com/pycqa/isort
     rev: 5.13.2
     hooks:

Makefile

@@ -5,6 +5,7 @@ MAKEFLAGS += --no-print-directory
 help:
 	@echo "Available targets for Felis:"
 	@echo "  build    - Build the package"
+	@echo "  deps     - Install dependencies"
 	@echo "  docs     - Generate the documentation"
 	@echo "  check    - Run pre-commit checks"
 	@echo "  test     - Run tests"
@@ -41,6 +42,7 @@ mypy:
 
 all:
 	@$(MAKE) build
+	@$(MAKE) deps
 	@$(MAKE) docs
 	@$(MAKE) check
 	@$(MAKE) test

docs/changes/DM-46130.feature.rst

@@ -0,0 +1,2 @@
+Add tools for comparing schemas in the `diff` module.
+These can be run from the command line using `felis diff`.

docs/dev/internals.rst

@@ -7,6 +7,10 @@ Python API
 .. automodapi:: felis.datamodel
    :include-all-objects:
 
+.. automodapi:: felis.diff
+   :include-all-objects:
+   :no-inheritance-diagram:
+
 .. automodapi:: felis.db.dialects
    :include-all-objects:
    :no-inheritance-diagram:

mypy.ini

@@ -10,7 +10,7 @@ plugins = pydantic.mypy
 [mypy-sqlalchemy.*]
 ignore_missing_imports = True
 
-[mypy-pyld.*]
+[mypy-deepdiff.*]
 ignore_missing_imports = True
 
 [mypy-felis.*]

pyproject.toml

@@ -22,13 +22,16 @@ classifiers = [
 ]
 keywords = ["lsst"]
 dependencies = [
-    "astropy >= 4",
-    "sqlalchemy >= 1.4",
-    "click >= 7",
-    "pyyaml >= 6",
-    "pydantic >= 2, < 3",
-    "lsst-utils",
-    "lsst-resources"
+    "alembic",
+    "astropy",
+    "click",
+    "deepdiff",
+    "lsst-resources @ git+https://github.com/lsst/resources@main",
+    "lsst-utils @ git+https://github.com/lsst/utils@main",
+    "numpy",
+    "pydantic >=2,<3",
+    "pyyaml",
+    "sqlalchemy"
 ]
 requires-python = ">=3.11.0"
 dynamic = ["version"]

python/felis/__init__.py

@@ -19,4 +19,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+from .datamodel import Schema
+from .db.schema import create_database
+from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
+from .metadata import MetaDataBuilder
 from .version import *

python/felis/cli.py

@@ -34,7 +34,9 @@
 
 from . import __version__
 from .datamodel import Schema
+from .db.schema import create_database
 from .db.utils import DatabaseContext, is_mock_url
+from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
 from .metadata import MetaDataBuilder
 from .tap import Tap11Base, TapLoadingVisitor, init_tables
 from .tap_schema import DataLoader, TableManager
@@ -493,5 +495,63 @@ def validate(
         raise click.exceptions.Exit(rc)
 
 
+@cli.command(
+    "diff",
+    help="""
+    Compare two schemas or a schema and a database for changes
+
+    Examples:
+
+      felis diff schema1.yaml schema2.yaml
+
+      felis diff -c alembic schema1.yaml schema2.yaml
+
+      felis diff --engine-url sqlite:///test.db schema.yaml
+    """,
+)
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option(
+    "-c",
+    "--comparator",
+    type=click.Choice(["alembic", "deepdiff"], case_sensitive=False),
+    help="Comparator to use for schema comparison",
+    default="deepdiff",
+)
+@click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
+@click.argument("files", nargs=-1, type=click.File())
+@click.pass_context
+def diff(
+    ctx: click.Context,
+    engine_url: str | None,
+    comparator: str,
+    error_on_change: bool,
+    files: Iterable[IO[str]],
+) -> None:
+    schemas = [
+        Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]}) for file in files
+    ]
+
+    diff: SchemaDiff
+    if len(schemas) == 2 and engine_url is None:
+        if comparator == "alembic":
+            db_context = create_database(schemas[0])
+            assert isinstance(db_context.engine, Engine)
+            diff = DatabaseDiff(schemas[1], db_context.engine)
+        else:
+            diff = FormattedSchemaDiff(schemas[0], schemas[1])
+    elif len(schemas) == 1 and engine_url is not None:
+        engine = create_engine(engine_url)
+        diff = DatabaseDiff(schemas[0], engine)
+    else:
+        raise click.ClickException(
+            "Invalid arguments - provide two schemas or a schema and a database engine URL"
+        )
+
+    diff.print()
+
+    if diff.has_changes and error_on_change:
+        raise click.ClickException("Schema was changed")
+
+
 if __name__ == "__main__":
     cli()

python/felis/db/schema.py

@@ -0,0 +1,63 @@
+"""Database utilities for Felis schemas."""
+
+# This file is part of felis.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from sqlalchemy import Engine, create_engine
+
+from ..datamodel import Schema
+from ..metadata import MetaDataBuilder
+from .utils import DatabaseContext
+
+__all__ = ["create_database"]
+
+
+def create_database(schema: Schema, engine_or_url_str: Engine | str | None = None) -> DatabaseContext:
+    """
+    Create a database from the specified `Schema`.
+
+    Parameters
+    ----------
+    schema
+        The schema to create.
+    engine_or_url_str
+        The SQLAlchemy engine or URL to use for database creation.
+        If None, an in-memory SQLite database will be created.
+
+    Returns
+    -------
+    `DatabaseContext`
+        The database context object.
+    """
+    if engine_or_url_str is not None:
+        engine = (
+            engine_or_url_str if isinstance(engine_or_url_str, Engine) else create_engine(engine_or_url_str)
+        )
+    else:
+        engine = create_engine("sqlite:///:memory:")
+    apply_schema = False if engine.url.drivername == "sqlite" else True
+    metadata = MetaDataBuilder(
+        schema, apply_schema_to_metadata=apply_schema, apply_schema_to_tables=apply_schema
+    ).build()
+    ctx = DatabaseContext(metadata, engine)
+    ctx.initialize()
+    ctx.create_all()
+    return ctx