Merge pull request #54 from lsst-dm/tickets/DM-47507

DM-47507: Add workflow for automatically generating migration scripts

.github/workflows/migrate.yaml

@@ -0,0 +1,112 @@
+name: Generate migration scripts
+
+on:
+  repository_dispatch:
+    types: [migration]
+  workflow_dispatch:
+    inputs:
+      branch_name:
+        description: 'Branch name in sdm_schemas to migrate'
+        required: true
+      commit_sha:
+        description: 'Commit SHA in sdm_schemas to migrate'
+        required: true
+
+jobs:
+  generate-migration:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout consdb repo
+        uses: actions/checkout@v4
+        with:
+          path: consdb
+          fetch-depth: 0
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12.7'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip uv
+          uv pip install --system lsst-felis testing.postgresql alembic sqlalchemy pyyaml black psycopg2-binary
+
+      - name: Determine branch name and commit SHA
+        run: |
+          if [ "${{ github.event_name }}" == "repository_dispatch" ]; then
+            echo "BRANCH_NAME=${{ github.event.client_payload.branch_name }}" >> $GITHUB_ENV
+            echo "COMMIT_SHA=${{ github.event.client_payload.commit_sha }}" >> $GITHUB_ENV
+          elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "BRANCH_NAME=${{ github.event.inputs.branch_name }}" >> $GITHUB_ENV
+            echo "COMMIT_SHA=${{ github.event.inputs.commit_sha }}" >> $GITHUB_ENV
+          fi
+          echo "Branch name: ${{ env.BRANCH_NAME }}"
+          echo "Commit SHA: ${{ env.COMMIT_SHA }}"
+          if [ -z "${{ env.BRANCH_NAME }}" ] || [ -z "${{ env.COMMIT_SHA }}" ]; then
+            echo "Error: Branch name and commit SHA must be provided." >&2
+            exit 1
+          fi
+
+      - name: Check branch name
+        run: |
+          if [[ ! "${{ env.BRANCH_NAME }}" =~ ^tickets/DM-[0-9]+(-[a-zA-Z0-9-]*)?$ ]]; then
+            echo "Bad branch name: ${{ env.BRANCH_NAME }}" >&2
+            echo "Error: Migrations are only generated for branches matching the pattern 'tickets/DM-[number](-[alphanumeric-]*)'." >&2
+            exit 1
+          fi
+
+      - name: Set ticket name
+        run: |
+          TICKET_NAME=$(echo ${{ env.BRANCH_NAME }} | sed -E 's/tickets\/(DM-[0-9]+).*/\1/')
+          echo "TICKET_NAME=${TICKET_NAME}" >> $GITHUB_ENV
+          if [[ ! "${{ env.TICKET_NAME }}" =~ ^DM-[0-9]+$ ]]; then
+            echo "Error: TICKET_NAME does not match the expected pattern 'DM-[number]'." >&2
+            exit 1
+          fi
+
+      - name: Checkout sdm_schemas repo
+        uses: actions/checkout@v4
+        with:
+          repository: lsst/sdm_schemas
+          ref: ${{ env.COMMIT_SHA }}
+          path: sdm_schemas
+
+      - name: Set sdm_schemas path in environment
+        run: |
+          echo "SDM_SCHEMAS_DIR=${GITHUB_WORKSPACE}/sdm_schemas" >> $GITHUB_ENV
+
+      - name: Generate the migration scripts
+        working-directory: $GITHUB_WORKSPACE/consdb
+        run: |
+          python alembic-autogenerate.py ${{ env.TICKET_NAME }}
+          if git diff --quiet; then
+            echo "No changes detected."
+            exit 0
+          fi
+
+      - name: Create PR for the migration
+        id: cpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          path: consdb
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: "${{ env.TICKET_NAME }}: Migrate schema changes"
+          commit-message: Migrate schema changes from ${{ env.TICKET_NAME }}
+          body: |
+            This PR migrates schema changes from [${{ env.TICKET_NAME }}](https://ls.st/${{ env.TICKET_NAME }}) to the database.
+
+            ## Checklist
+
+            - [ ] Verified the automatically generated migration scripts
+          branch: ${{ env.BRANCH_NAME }}-migrate
+          base: main
+          draft: true
+          labels: migration
+
+      - name: Check PR outputs
+        if: ${{ steps.cpr.outputs.pull-request-number }}
+        run: |
+          echo "Pull Request Number: ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL: ${{ steps.cpr.outputs.pull-request-url }}"

alembic-autogenerate.py

@@ -14,9 +14,7 @@
 # 3. Heed the message at the end to revise your auto-generated code as needed.
 #
 
-import glob
 import os
-import re
 import sys
 
 from alembic.config import Config
@@ -42,7 +40,7 @@
 
 # Loop over each of the instruments
 pattern = os.environ["SDM_SCHEMAS_DIR"] + "/yml/cdb_*.yaml"
-instruments = [re.search(r"cdb_(.+)\.yaml$", file).group(1) for file in glob.glob(pattern)]
+instruments = ["latiss", "lsstcomcam", "lsstcomcamsim"]
 for instrument in instruments:
     # Set up a temporary PostgreSQL instance using testing.postgresql
     with setup_postgres_test_db() as instance:

alembic/env.py

@@ -48,6 +48,14 @@ def include_name(name, type_, parent_names):
         return True
 
 
+def include_object(object, name, type_, reflected, compare_to):
+    if type_ == "table" and name in ["ccdvisit1", "visit1"]:
+        logger.info(f"Excluding table {object.schema}.{name}")
+        return False
+    else:
+        return True
+
+
 # add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
@@ -80,6 +88,7 @@ def run_migrations_offline() -> None:
         dialect_opts={"paramstyle": "named"},
         include_schemas=True,
         include_name=include_name,
+        include_object=include_object,
         version_table=f"{schema_name}_version",
         version_table_schema="cdb",
     )
@@ -107,6 +116,7 @@ def run_migrations_online() -> None:
             target_metadata=target_metadata,
             include_schemas=True,
             include_name=include_name,
+            include_object=include_object,
             version_table=f"{schema_name}_version",
             version_table_schema="cdb",
         )