Merge pull request #11 from lsst-dm/tickets/DM-36468

DM-36468: Enable db-auth.yaml credentials use

Author: andy-slac

.github/workflows/build.yaml

@@ -8,24 +8,28 @@ on:
 
 jobs:
   build_and_test:
-    # runs-on: ubuntu-latest
-    # Need >= 20.04 for modern sqlite. Can switch to latest when
-    # github change
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        with:
+          # Need to clone everything for the git tags.
+          fetch-depth: 0
 
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: 3.8
+          python-version: ${{ matrix.python-version }}
           channels: conda-forge,defaults
           channel-priority: strict
           show-channel-urls: true
 
       - name: Update pip/wheel infrastructure
         shell: bash -l {0}
         run: |
-          conda install -y -q pip wheel
+          conda install -y -q "pip<22" wheel
 
       - name: Install dependencies
         shell: bash -l {0}

.github/workflows/formatting.yaml

@@ -1,4 +1,4 @@
-name: check Python formatting
+name: Check Python formatting
 
 on:
   push:
@@ -7,21 +7,5 @@ on:
   pull_request:
 
 jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Install isort and black
-        run: pip install isort black
-
-      - name: Run isort
-        run: isort --diff --check-only python/ tests/
-
-      - name: Run black
-        run: black --check --verbose --diff python/ tests/
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/formatting.yaml@main

.github/workflows/lint.yaml

@@ -1,22 +1,11 @@
 name: lint
 
 on:
-  - push
-  - pull_request
+  push:
+    branches:
+      - main
+  pull_request:
 
 jobs:
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Install
-        run: pip install -r <(curl https://raw.githubusercontent.com/lsst/linting/main/requirements.txt)
-
-      - name: Run linter
-        run: flake8
+  call-workflow:
+    uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main

.github/workflows/mypy.yaml

@@ -0,0 +1,42 @@
+name: Run mypy
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10.6"
+          cache: "pip"
+
+      - name: Install
+        run: pip install mypy pydantic
+
+      - name: Install types dependencies
+        run: |
+          if [ -e types.txt ]
+          then
+            pip install -r types.txt
+          fi
+      - name: Install lsst dependencies
+        run: |
+          if [ -e requirements.txt ]
+          then
+             pip install -r requirements.txt
+          fi
+      - name: Run mypy
+        run: |
+          prefix="python/lsst"
+          echo "__all__ = ['__version__']" > "${prefix}/daf/butler_migrate/version.py"
+          echo "__version__ = '0.0.1'" >> "${prefix}/daf/butler_migrate/version.py"
+          mypy "${prefix}"

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v4.3.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
@@ -13,9 +13,13 @@ repos:
         # supported by your project here, or alternatively use
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
-        language_version: python3.8
+        language_version: python3.10
   - repo: https://github.com/pycqa/isort
     rev: 5.10.1
     hooks:
       - id: isort
         name: isort (python)
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8

bin.src/SConscript

@@ -1,7 +1,65 @@
+[build-system]
+requires = ["setuptools", "lsst-versions >= 1.3.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "lsst-daf-butler-migrate"
+description = "Tooling for migrating schemas in butler repositories."
+license = {text = "GPLv3+ License"}
+readme = "README.md"
+authors = [
+    {name="Rubin Observatory Data Management", email="[email protected]"},
+]
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Topic :: Scientific/Engineering :: Astronomy",
+]
+keywords = ["lsst"]
+dependencies = [
+    "lsst-utils",
+    "lsst-resources",
+    "lsst-daf-butler",
+    "click",
+    "sqlalchemy",
+    "alembic"
+]
+dynamic = ["version"]
+
+[project.urls]
+"Homepage" = "https://github.com/lsst-dm/daf_butler_migrate"
+
+[project.optional-dependencies]
+test = [
+    "pytest >= 3.2",
+    "flake8 >= 3.7.5",
+    "pytest-flake8 >= 1.0.4",
+    "pytest-openfiles >= 0.5.0"
+]
+
+[tool.setuptools.packages.find]
+where = ["python"]
+
+[tool.setuptools]
+zip-safe = true
+license-files = ["COPYRIGHT", "LICENSE"]
+
+[tool.setuptools.package-data]
+"lsst.daf.butler_migrate" = []
+
+[tool.setuptools.dynamic]
+version = { attr = "lsst_versions.get_lsst_version" }
+
 [tool.black]
 line-length = 110
-target-version = ["py38"]
+target-version = ["py310"]
 
 [tool.isort]
 profile = "black"
 line_length = 110
+
+[tool.lsst_versions]
+write_to = "python/lsst/daf/butler_migrate/version.py"

pyproject.toml

@@ -28,8 +28,7 @@
 
 import sqlalchemy
 from alembic.runtime.migration import MigrationContext
-from lsst.daf.butler import ButlerConfig
-from lsst.daf.butler.core.repoRelocation import replaceRoot
+from lsst.daf.butler import ButlerConfig, RegistryConfig
 
 from . import revision
 
@@ -59,7 +58,7 @@ class Database:
     dimensions_config_manager = "dimensions-config"
     """Name of the special dimensions-config pseudo-manager (`str`)"""
 
-    def __init__(self, db_url: str, schema: Optional[str] = None):
+    def __init__(self, db_url: sqlalchemy.engine.url.URL, schema: Optional[str] = None):
         self._db_url = db_url
         self._schema = schema
 
@@ -74,11 +73,10 @@ def from_repo(cls, repo: str) -> Database:
             "butler.yaml" file.
         """
         butlerConfig = ButlerConfig(repo)
-        if "root" in butlerConfig:
-            butlerRoot = butlerConfig["root"]
-        else:
-            butlerRoot = butlerConfig.configDir
-        db_url = replaceRoot(butlerConfig["registry", "db"], butlerRoot)
+        registryConfig = RegistryConfig(butlerConfig)
+        butlerRoot = butlerConfig.get("root", butlerConfig.configDir)
+        registryConfig.replaceRoot(butlerRoot)
+        db_url = registryConfig.connectionString
         schema: Optional[str] = None
         try:
             schema = butlerConfig["registry", "namespace"]
@@ -91,7 +89,7 @@ def from_repo(cls, repo: str) -> Database:
     @property
     def db_url(self) -> str:
         """URL for registry database (`str`)"""
-        return self._db_url
+        return str(self._db_url)
 
     @property
     def schema(self) -> Optional[str]:

python/lsst/daf/butler_migrate/database.py

@@ -131,7 +131,9 @@ def _migrate_revision_one_shot(mig_path: str, tree_name: str, manager_class: str
     assert len(bases) == 1
 
     # Base revision has a random ID but its branch label is "<tree>"
-    branches = scripts.get_revision(bases[0]).branch_labels
+    revision_script = scripts.get_revision(bases[0])
+    assert revision_script is not None, "Script for a known base must exist"
+    branches = revision_script.branch_labels
     assert len(branches) == 1
     manager = branches.pop()
 

python/lsst/daf/butler_migrate/script/migrate_revision.py

@@ -54,6 +54,7 @@ def migrate_trees(mig_path: str, verbose: bool, one_shot: bool) -> None:
     bases_map: Dict[str, Script] = {}
     for name in bases:
         revision = scripts.get_revision(name)
+        assert revision is not None, "Script for a known base must exist"
         # Base revision has a random ID but its branch label is "<tree>"
         branches = revision.branch_labels
         if not branches:
@@ -93,6 +94,7 @@ def _migrate_trees_one_shot(mig_path: str, verbose: bool) -> None:
             if bases:
                 assert len(bases) == 1
                 revision = scripts.get_revision(bases[0])
+                assert revision is not None, "Script for a known base must exist"
                 print(revision.log_entry)
         else:
             print(entry)

python/lsst/daf/butler_migrate/script/migrate_trees.py

@@ -1,5 +1,5 @@
 click >7.0
-sqlalchemy >= 1.3
+sqlalchemy >= 1.4
 alembic
 lsst-daf-butler
 lsst-utils

requirements.txt

@@ -1,39 +1,3 @@
-[metadata]
-name = lsst_butler_migrate
-description = Tooling for migrating schemas in butler repositories.
-author = Rubin Observatory Data Management
-url = https://github.com/lsst-dm/daf_butler_migrate
-classifiers =
-    Intended Audience :: Science/Research
-    License :: OSI Approved ::  GNU General Public License v3 or later (GPLv3+)
-    Operating System :: OS Independent
-    Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
-    Topic :: Scientific/Engineering :: Astronomy
-
-[options]
-zip_safe = True
-package_dir=
-    =python
-packages=find:
-setup_requires =
-  setuptools >=46.0
-install_requires =
-  click >7.0
-  sqlalchemy >= 1.3
-  alembic >= 1.7
-  lsst-resources
-  lsst-utils
-  lsst-daf-butler
-tests_require =
-  pytest >= 3.2
-  flake8 >= 3.7.5
-  pytest-flake8 >= 1.0.4
-  pytest-openfiles >= 0.5.0
-
-[options.packages.find]
-where=python
-
 [flake8]
 max-line-length = 110
 max-doc-length = 79
@@ -46,7 +10,3 @@ exclude =
   tests/.tests
 per-file-ignores =
   migrations/**:F401,W505
-
-[tool:pytest]
-addopts = --flake8
-flake8-ignore = E133 E226 E228 N802 N803 N806 N812 N813 N815 N816 W503 E203