From c662e9ad4b99400d7651793a740434456735e4ae Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Mon, 16 Dec 2024 16:55:31 -0500 Subject: [PATCH 1/9] Add projection finder --- .../psf/service/backend/projection_finders.py | 275 ++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 python/lsst/psf/service/backend/projection_finders.py diff --git a/python/lsst/psf/service/backend/projection_finders.py b/python/lsst/psf/service/backend/projection_finders.py new file mode 100644 index 0000000..daea78a --- /dev/null +++ b/python/lsst/psf/service/backend/projection_finders.py @@ -0,0 +1,275 @@ +# This file is part of psf_service_backend. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import annotations + +__all__ = ( + "Chain", + "MatchDatasetTypeName", + "ProjectionFinder", + "ReadComponents", + "TryComponentParents", + "UseSkyMap", +) + +import re +from abc import ABC, abstractmethod +from collections.abc import Iterable +from typing import cast + +from lsst.afw.geom import SkyWcs +from lsst.daf.butler import Butler, DatasetRef +from lsst.geom import Box2I +from lsst.skymap import BaseSkyMap + + +class ProjectionFinder(ABC): + """An interface for objects that can find the WCS and bounding box of a + butler dataset. + + Notes + ----- + Concrete `ProjectionFinder` implementations are intended to be composed to + define rules for how to find this projection information for a particular + dataset type; a finder that cannot handle a particular `DatasetRef` should + implement `find_projection` to return `None`, and implementations that + compose (e.g. `Chain`) can use this to decide when to try another nested + finder. + """ + + @abstractmethod + def find_projection(self, ref: DatasetRef, butler: Butler) -> tuple[SkyWcs, Box2I] | None: + """Run the finder on the given dataset with the given butler. + + Parameters + ---------- + ref : `DatasetRef` + Fully-resolved reference to the dataset. + butler : `Butler` + Butler client to use for reads. Need not support writes, and any + default search collections will be ignored. + + Returns + ------- + wcs : `SkyWcs` (only if result is not `None`) + Mapping from sky to pixel coordinates for this dataset. + bbox : `Box2I` (only if result is not `None`) + Bounding box of the image dataset (or an image closely associated + with the dataset) in pixel coordinates. + """ + raise NotImplementedError() + + def __call__(self, ref: DatasetRef, butler: Butler) -> tuple[SkyWcs, Box2I]: + """A thin wrapper around `find_projection` that raises `LookupError` + when no projection information was found. + + Parameters + ---------- + ref : `DatasetRef` + Fully-resolved reference to the dataset. + butler : `Butler` + Butler client to use for reads. Need not support writes, and any + default search collections will be ignored. + + Returns + ------- + wcs : `SkyWcs` + Mapping from sky to pixel coordinates for this dataset. + bbox : `Box2I` + Bounding box of the image dataset (or an image closely associated + with the dataset) in pixel coordinates. + """ + result = self.find_projection(ref, butler) + if result is None: + raise LookupError(f"No way to obtain WCS and bounding box information for ref {ref}.") + return result + + @staticmethod + def make_default() -> ProjectionFinder: + """Return a concrete finder appropriate for most pipelines. + + Returns + ------- + finder : `ProjectionFinder` + A finder that prefers to read, use, and cache a skymap when the + data ID includes tract or patch, and falls back to reading the WCS + and bbox from the dataset itself (or its parent, if the dataset is + a component). + """ + return TryComponentParents( + Chain( + UseSkyMap(), + ReadComponents(), + ) + ) + + +class ReadComponents(ProjectionFinder): + """A `ProjectionFinder` implementation that reads ``wcs`` and ``bbox`` + from datasets that have them (e.g. ``Exposure``). + + Notes + ----- + This should usually be the final finder attempted in any chain; it's the + one most likely to work, but in many cases will not be the most efficient + or yield the most accurate WCS. + """ + + def find_projection(self, ref: DatasetRef, butler: Butler) -> tuple[SkyWcs, Box2I] | None: + # Docstring inherited. + if {"wcs", "bbox"}.issubset(ref.datasetType.storageClass.allComponents().keys()): + wcs = butler.get(ref.makeComponentRef("wcs")) + bbox = butler.get(ref.makeComponentRef("bbox")) + return wcs, bbox + return None + + +class TryComponentParents(ProjectionFinder): + """A composite `ProjectionFinder` that walks from component dataset to its + parent composite until its nested finder succeeds. + + Parameters + ---------- + nested : `ProjectionFinder` + Nested finder to delegate to. + + Notes + ----- + This is a good choice for the outermost composite finder, so that the same + sequence of nested rules are applied to each level of the + component-composite tree. + """ + + def __init__(self, nested: ProjectionFinder): + self._nested = nested + + def find_projection(self, ref: DatasetRef, butler: Butler) -> tuple[SkyWcs, Box2I] | None: + # Docstring inherited. + while True: + if (result := self._nested.find_projection(ref, butler)) is not None: + return result + if ref.isComponent(): + ref = ref.makeCompositeRef() + else: + return None + + +class UseSkyMap(ProjectionFinder): + """A `ProjectionFinder` implementation that reads and caches + `lsst.skymap.BaseSkyMap` instances, allowing projections for coadds to be + found without requiring I/O for each one. + + Parameters + ---------- + dataset_type_name : `str`, optional + Name of the dataset type used to load `BaseSkyMap` instances. + collections : `Iterable` [ `str` ] + Collection search path for skymap datasets. + + Notes + ----- + This finder assumes any dataset with ``patch`` or ``tract`` dimensions + should get its WCS and bounding box from the skymap, and that datasets + without these dimensions never should (i.e. `find_projection` will return + `None` for these). + + `BaseSkyMap` instances are never removed from the cache after being loaded; + we expect the number of distinct skymaps to be very small. + """ + + def __init__(self, dataset_type_name: str = "skyMap", collections: Iterable[str] = ("skymaps",)): + self._dataset_type_name = dataset_type_name + self._collections = tuple(collections) + self._cache: dict[str, BaseSkyMap] = {} + + def find_projection(self, ref: DatasetRef, butler: Butler) -> tuple[SkyWcs, Box2I] | None: + # Docstring inherited. + if "tract" in ref.dataId.dimensions: + assert "skymap" in ref.dataId.dimensions, "Guaranteed by expected dimension schema." + if (skymap := self._cache.get(cast(str, ref.dataId["skymap"]))) is None: + skymap = butler.get( + self._dataset_type_name, skymap=ref.dataId["skymap"], collections=self._collections + ) + tractInfo = skymap[ref.dataId["tract"]] + if "patch" in ref.dataId.dimensions: + patchInfo = tractInfo[ref.dataId["patch"]] + return patchInfo.wcs, patchInfo.outer_bbox + else: + return tractInfo.wcs, tractInfo.bbox + return None + + +class Chain(ProjectionFinder): + """A composite `ProjectionFinder` that attempts each finder in a sequence + until one succeeds. + + Parameters + ---------- + *nested : `ProjectionFinder` + Nested finders to delegate to, in order. + """ + + def __init__(self, *nested: ProjectionFinder): + self._nested = tuple(nested) + + def find_projection(self, ref: DatasetRef, butler: Butler) -> tuple[SkyWcs, Box2I] | None: + # Docstring inherited. + for f in self._nested: + if (result := f.find_projection(ref, butler)) is not None: + return result + return None + + +class MatchDatasetTypeName(ProjectionFinder): + """A composite `ProjectionFinder` that delegates to different nested + finders based on whether the dataset type name matches a regular + expression. + + Parameters + ---------- + regex : `str` + Regular expression the dataset type name must match (in full). + on_match : `ProjectionFinder`, optional + Finder to try when the match succeeds, or `None` to return `None`. + otherwise : `ProjectionFinder`, optional + Finder to try when the match does not succeed, or `None` to return + `None`. + """ + + def __init__( + self, + regex: str, + on_match: ProjectionFinder | None = None, + otherwise: ProjectionFinder | None = None, + ): + self._regex = re.compile(regex) + self._on_match = on_match + self._otherwise = otherwise + + def find_projection(self, ref: DatasetRef, butler: Butler) -> tuple[SkyWcs, Box2I] | None: + # Docstring inherited. + if self._regex.match(ref.datasetType.name): + if self._on_match is not None: + return self._on_match.find_projection(ref, butler) + else: + if self._otherwise is not None: + return self._otherwise.find_projection(ref, butler) + return None From ab0d145c5e5daeb5c421fd855d1064978eac3328 Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Mon, 16 Dec 2024 16:56:44 -0500 Subject: [PATCH 2/9] Introduce psf service backend --- python/lsst/psf/service/backend/__init__.py | 2 + .../service/backend/_psf_service_backend.py | 338 ++++++++++++++++++ 2 files changed, 340 insertions(+) create mode 100644 python/lsst/psf/service/backend/_psf_service_backend.py diff --git a/python/lsst/psf/service/backend/__init__.py b/python/lsst/psf/service/backend/__init__.py index 7385703..a5a33cc 100644 --- a/python/lsst/psf/service/backend/__init__.py +++ b/python/lsst/psf/service/backend/__init__.py @@ -19,4 +19,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from . import projection_finders +from ._psf_service_backend import * from .version import * # Generated by sconsUtils diff --git a/python/lsst/psf/service/backend/_psf_service_backend.py b/python/lsst/psf/service/backend/_psf_service_backend.py new file mode 100644 index 0000000..6c36e88 --- /dev/null +++ b/python/lsst/psf/service/backend/_psf_service_backend.py @@ -0,0 +1,338 @@ +# This file is part of psf_service_backend. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from __future__ import annotations + +__all__ = ("PsfServiceBackend", "PsfExtraction") + +import dataclasses +from uuid import UUID, uuid4 +from collections.abc import Sequence + +from lsst.afw.image import ImageD +from lsst.daf.base import PropertyList +from lsst.daf.butler import Butler, DataId, DatasetRef +from lsst.resources import ResourcePath, ResourcePathExpression +from .projection_finders import ProjectionFinder +import lsst.geom as geom + + +@dataclasses.dataclass +class PsfExtraction: + """A struct that stores the extracted PSF model at a point.""" + + psf_image: ImageD + """The PSF image itself.""" + + metadata: PropertyList + """Additional FITS metadata about the PSF extraction process.""" + + origin_ref: DatasetRef + """Fully-resolved reference to the dataset the PSF is from.""" + + def write_fits(self, path: str) -> None: + """Write the PSF image to a FITS file. + + Parameters + ---------- + path : `str` + Local path to the file. + """ + self.psf_image.writeFits(fileName=path, metadata=self.metadata) + + +class PsfServiceBackend: + """High-level interface to the PSF service backend. + + This backend can retrieve the PSF model from various LSST image datasets, + for example: + - `calexp` (Per-detector processed exposure), + - `deepCoadd_calexp` (calibrated coadd exposures), + - `deepDiff_differenceExp` (difference image produced in AP pipeline). + + Any dataset type that has a `getPsf()` method should be supported. Refer to + the LSST Data Management code on GitHub (https://github.com/lsst) for more + information on dataset types. + + Parameters + ---------- + butler : `Butler` + Butler that retrieves images of various types from the LSST Science + Pipelines data repository. + projection_finder : `ProjectionFinder` + Object used to obtain WCS for butler datasets, allowing RA/Dec to pixel + conversions. + output_root : `ResourcePathExpression` + Root of output file URIs. The final PSF FITS file will be placed here. + temporary_root : `ResourcePathExpression`, optional + Local filesystem root for writing temporary files before transferring + to `output_root`. + """ + + def __init__( + self, + butler: Butler, + projection_finder: ProjectionFinder, + output_root: ResourcePathExpression, + temporary_root: ResourcePathExpression | None = None, + ): + self.butler = butler + self.projection_finder = projection_finder + self.output_root = ResourcePath(output_root, forceAbsolute=True, forceDirectory=True) + self.temporary_root = ( + ResourcePath(temporary_root, forceDirectory=True) if temporary_root is not None else None + ) + + butler: Butler + projection_finder: ProjectionFinder + output_root: ResourcePath + temporary_root: ResourcePath | None + + def process_ref(self, ra: float, dec: float, ref: DatasetRef) -> ResourcePath: + """Retrieve and write a PSF image from a fully-resolved `DatasetRef`. + + Parameters + ---------- + ra, dec : `float` + Right Ascension and Declination of the point (in degrees) where the + PSF should be evaluated. + ref : `DatasetRef` + Fully-resolved reference to a dataset (e.g., `calexp`, + `deepCoadd_calexp`, `goodSeeingDiff_differenceExp`). + + Returns + ------- + uri : `ResourcePath` + Full path to the extracted PSF image file. + """ + psf_result = self.extract_ref(ra, dec, ref) + return self.write_fits(psf_result) + + def process_uuid( + self, + ra: float, + dec: float, + uuid: UUID, + *, + component: str | None = None, + ) -> ResourcePath: + """Retrieve and write a PSF image from a dataset identified by its + UUID. + + Parameters + ---------- + ra, dec : `float` + RA/Dec of the point where the PSF should be evaluated (in degrees). + uuid : `UUID` + Unique ID of the dataset (e.g., a `calexp`). + component : `str`, optional + If not None, read this component instead of the composite dataset. + + Returns + ------- + uri : `ResourcePath` + Full path to the extracted PSF image file. + """ + psf_result = self.extract_uuid(ra, dec, uuid, component=component) + return self.write_fits(psf_result) + + def process_search( + self, + ra: float, + dec: float, + dataset_type_name: str, + data_id: DataId, + collections: Sequence[str], + ) -> ResourcePath: + """Retrieve and write a PSF image from a dataset identified by + (dataset type, data ID, collection). + + Parameters + ---------- + ra, dec : `float` + RA/Dec of the point where the PSF should be evaluated (in degrees). + dataset_type_name : `str` + Name of the butler dataset (e.g. "calexp", "deepCoadd_calexp", + "goodSeeingDiff_differenceExp"). + data_id : `dict` or `DataCoordinate` + Data ID used to find the dataset (e.g. {"visit": 12345, + "detector": 42}). + collections : `Sequence[str]` + Collections to search for the dataset. + + Returns + ------- + uri : `ResourcePath` + Full path to the extracted PSF image file. + """ + psf_result = self.extract_search(ra, dec, dataset_type_name, data_id, collections) + return self.write_fits(psf_result) + + def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: + """Extract a PSF image from a fully-resolved `DatasetRef`. + + Parameters + ---------- + ra, dec : `float` + RA/Dec of the point where the PSF should be evaluated (in degrees). + ref : `DatasetRef` + Fully-resolved dataset reference. + + Returns + ------- + psf_extraction : `PsfExtraction` + Struct containing the PSF image, metadata and the origin reference. + + Raises + ------ + ValueError + If `ref.id` is not resolved or if the dataset does not contain a + PSF. + """ + if ref.id is None: + raise ValueError(f"A resolved DatasetRef with a valid ID is required; got {ref}.") + + # Obtain WCS from the dataset and convert RA/Dec to pixels. + wcs, _ = self.projection_finder(ref, self.butler) + point_sky = geom.SpherePoint(geom.Angle(ra, geom.degrees), geom.Angle(dec, geom.degrees)) + point_pixel = wcs.skyToPixel(point_sky) + + # Get the image from the butler and extract the PSF model. + image = self.butler.get(ref) + + if not hasattr(image, "getPsf"): + raise ValueError( + f"The dataset {ref.datasetType.name} with ID {ref.id} does not have a `getPSF()` method" + ) + + psf = image.getPsf() + if psf is None: + raise ValueError(f"No PSF found in dataset {ref.datasetType.name} with ID {ref.id}.") + + # Compute the PSF kernel image at the given point using the PSF model. + psf_image = psf.computeKernelImage(point_pixel) + + # Create FITS metadata. + metadata = PropertyList() + metadata.set("BTLRUUID", ref.id.hex, "Butler dataset UUID from which this PSF was extracted.") + metadata.set( + "BTLRNAME", ref.datasetType.name, "Butler dataset type name from which this PSF was extracted." + ) + metadata.set("PSFRA", ra, "RA of the PSF evaluation point (deg).") + metadata.set("PSFDEC", dec, "Dec of the PSF evaluation point (deg).") + + for n, (k, v) in enumerate(ref.dataId.required.items()): + metadata.set(f"BTLRK{n:03}", k, f"Name of dimension {n} in the data ID.") + metadata.set(f"BTLRV{n:03}", v, f"Value of dimension {n} in the data ID.") + + return PsfExtraction( + psf_image=psf_image, + metadata=metadata, + origin_ref=ref, + ) + + def extract_uuid( + self, ra: float, dec: float, uuid: UUID, *, component: str | None = None + ) -> PsfExtraction: + """Extract a PSF image from a dataset identified by its UUID. + + Parameters + ---------- + ra, dec : `float` + RA/Dec (deg) of the PSF evaluation point. + uuid : `UUID` + Unique dataset identifier. + component : `str`, optional + If not None, read this component of the dataset. + + Returns + ------- + psf_extraction : `PsfExtraction` + Struct containing the PSF image, metadata and the origin reference. + + Raises + ------ + LookupError + If no dataset is found with the given UUID. + """ + ref = self.butler.get_dataset(uuid) + if ref is None: + raise LookupError(f"No dataset found with UUID {uuid}.") + if component is not None: + ref = ref.makeComponentRef(component) + + return self.extract_ref(ra, dec, ref) + + def extract_search( + self, ra: float, dec: float, dataset_type_name: str, data_id: DataId, collections: Sequence[str] + ) -> PsfExtraction: + """Extract a PSF image from a dataset identified by (dataset type, + data ID, collection). + + Parameters + ---------- + ra, dec : `float` + RA/Dec (deg) of the PSF evaluation point. + dataset_type_name : `str` + Dataset type name of the image (e.g. "calexp", "deepCoadd_calexp", + "goodSeeingDiff_differenceExp"). + data_id : `DataId` + Data ID mapping used to locate the dataset. + collections : `Sequence[str]` + Collections to search for the dataset. + + Returns + ------- + psf_extraction : `PsfExtraction` + Struct containing the PSF image, metadata and the origin reference. + + Raises + ------ + LookupError + If no dataset is found with the given parameters. + """ + ref = self.butler.find_dataset(dataset_type_name, data_id, collections=collections) + if ref is None: + raise LookupError( + f"No {dataset_type_name} dataset found with data ID {data_id} in {collections}." + ) + return self.extract_ref(ra, dec, ref) + + def write_fits(self, psf_result: PsfExtraction) -> ResourcePath: + """Write a `PsfExtraction` to a FITS file in `output_root`. + + Parameters + ---------- + psf_result : `PsfExtraction` + The PSF extraction result to write. + + Returns + ------- + uri : `ResourcePath` + Full path to the extracted PSF file. + """ + output_uuid = uuid4() + remote_uri = self.output_root.join(output_uuid.hex + ".fits") + with ResourcePath.temporary_uri(prefix=self.temporary_root, suffix=".fits") as tmp_uri: + tmp_uri.parent().mkdir() + psf_result.write_fits(tmp_uri.ospath) + remote_uri.transfer_from(tmp_uri, transfer="copy") + return remote_uri From d79f613d52001a5ee1a199462badce912f1b34ee Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Mon, 16 Dec 2024 16:58:46 -0500 Subject: [PATCH 3/9] Add description to readme and modify ups table --- README.rst | 2 +- ups/psf_service_backend.table | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index dbe5903..4ee0f05 100644 --- a/README.rst +++ b/README.rst @@ -4,4 +4,4 @@ psf_service_backend ``psf_service_backend`` is a package in the `LSST Science Pipelines `_. -.. Add a brief (few sentence) description of what this package provides. +This backend generates the PSF representation as an image for a VO-compatible [PSF recreation and visualization support service](https://github.com/lsst-sqre/psf-service) at a specified point in any of several image types (calexp/PVI, coadd, difference image). \ No newline at end of file diff --git a/ups/psf_service_backend.table b/ups/psf_service_backend.table index 848450f..f27f437 100644 --- a/ups/psf_service_backend.table +++ b/ups/psf_service_backend.table @@ -1,8 +1,12 @@ # List EUPS dependencies of this package here. -# - Any package whose API is used directly should be listed explicitly. -# - Common third-party packages can be assumed to be recursively included by -# the "base" package. -setupRequired(base) +setupRequired(daf_butler) +setupRequired(sphgeom) +setupRequired(geom) +setupRequired(afw) +setupRequired(skymap) + +# For tests of retrieving PSFs from images in a butler repo. +setupOptional(testdata_image_cutouts) # The following is boilerplate for all packages. # See https://dmtn-001.lsst.io for details on LSST_LIBRARY_PATH. From 2b70da13e71ac119e3d455007c66d2e21d4f86fd Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Tue, 17 Dec 2024 17:09:02 -0500 Subject: [PATCH 4/9] Add unit test for psf service backend using test-med-1 data --- tests/test_psfServiceBackend.py | 87 +++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/test_psfServiceBackend.py diff --git a/tests/test_psfServiceBackend.py b/tests/test_psfServiceBackend.py new file mode 100644 index 0000000..5dd38cd --- /dev/null +++ b/tests/test_psfServiceBackend.py @@ -0,0 +1,87 @@ +# This file is part of psf_service_backend. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import tempfile +import unittest + +import lsst.daf.butler +import lsst.utils.tests +from lsst.psf.service.backend import PsfServiceBackend, projection_finders + + +class TestPsfServiceBackend(lsst.utils.tests.TestCase): + @classmethod + def setUpClass(cls): + try: + # Note: we are taking advantage of the exiting + # `testdata_image_cutouts` here, as it contains the necessary + # data to extract a PSF from and test the backend. + cls.data_dir = lsst.utils.getPackageDir("testdata_image_cutouts") + except LookupError: + raise unittest.SkipTest( + "PSFs must be extracted from the images stored in `testdata_image_cutouts` " + "which is not set up." + ) + + def setUp(self): + # Set up the butler using the test repository and collection. + repo = os.path.join(self.data_dir, "repo") + self.collection = "2.2i/runs/test-med-1/w_2022_03/DM-33223/20220118T193330Z" + self.butler = lsst.daf.butler.Butler(repo, collections=self.collection) + + # Example dataset type known to contain a PSF. + self.datasetType = "deepCoadd_calexp" + + # Example dataId within the test data repository. + self.dataId = {"patch": 24, "tract": 3828, "band": "r", "skymap": "DC2"} + + # RA/Dec within the image; we choose coordinates inside the dataset + # footprint. + self.ra = 56.6400770 + self.dec = -36.4492250 + + # For handling projections. + self.projectionFinder = projection_finders.ReadComponents() + + def test_extract_psf(self): + """Test PSF retrieval from a known dataset at a given RA/Dec.""" + ref = self.butler.registry.findDataset(self.datasetType, dataId=self.dataId) + if ref is None: + self.skipTest(f"No {self.datasetType} dataset found for dataId={self.dataId}.") + + with tempfile.TemporaryDirectory() as tempdir: + psfBackend = PsfServiceBackend(self.butler, self.projectionFinder, tempdir) + psfExtraction = psfBackend.extract_ref(self.ra, self.dec, ref) + + # Check that we got a valid PSF image. + self.assertIsNotNone(psfExtraction.psf_image, "No PSF image returned.") + psfImArray = psfExtraction.psf_image.array + self.assertGreater(psfImArray.size, 0, "PSF image is empty.") + self.assertNotEqual(psfImArray.sum(), 0.0, "PSF image appears to contain only zeros.") + + # Write out the result and confirm the file is produced. + resultUri = psfBackend.write_fits(psfExtraction) + self.assertTrue(os.path.exists(resultUri.ospath), "PSF FITS file was not created.") + + +if __name__ == "__main__": + unittest.main() From a6aaca1e8186d525011be4b48beee04401f55371 Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Thu, 19 Dec 2024 23:20:18 -0500 Subject: [PATCH 5/9] Ensure fully-qualified names in docstrings --- .../service/backend/_psf_service_backend.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/python/lsst/psf/service/backend/_psf_service_backend.py b/python/lsst/psf/service/backend/_psf_service_backend.py index 6c36e88..5be44e5 100644 --- a/python/lsst/psf/service/backend/_psf_service_backend.py +++ b/python/lsst/psf/service/backend/_psf_service_backend.py @@ -74,15 +74,15 @@ class PsfServiceBackend: Parameters ---------- - butler : `Butler` + butler : `lsst.daf.butler.Butler` Butler that retrieves images of various types from the LSST Science Pipelines data repository. projection_finder : `ProjectionFinder` Object used to obtain WCS for butler datasets, allowing RA/Dec to pixel conversions. - output_root : `ResourcePathExpression` + output_root : `lsst.resources.ResourcePathExpression` Root of output file URIs. The final PSF FITS file will be placed here. - temporary_root : `ResourcePathExpression`, optional + temporary_root : `lsst.resources.ResourcePathExpression`, optional Local filesystem root for writing temporary files before transferring to `output_root`. """ @@ -114,13 +114,13 @@ def process_ref(self, ra: float, dec: float, ref: DatasetRef) -> ResourcePath: ra, dec : `float` Right Ascension and Declination of the point (in degrees) where the PSF should be evaluated. - ref : `DatasetRef` + ref : `lsst.daf.butler.DatasetRef` Fully-resolved reference to a dataset (e.g., `calexp`, `deepCoadd_calexp`, `goodSeeingDiff_differenceExp`). Returns ------- - uri : `ResourcePath` + uri : `lsst.resources.ResourcePath` Full path to the extracted PSF image file. """ psf_result = self.extract_ref(ra, dec, ref) @@ -141,14 +141,14 @@ def process_uuid( ---------- ra, dec : `float` RA/Dec of the point where the PSF should be evaluated (in degrees). - uuid : `UUID` + uuid : `uuid.UUID` Unique ID of the dataset (e.g., a `calexp`). component : `str`, optional If not None, read this component instead of the composite dataset. Returns ------- - uri : `ResourcePath` + uri : `lsst.resources.ResourcePath` Full path to the extracted PSF image file. """ psf_result = self.extract_uuid(ra, dec, uuid, component=component) @@ -172,15 +172,15 @@ def process_search( dataset_type_name : `str` Name of the butler dataset (e.g. "calexp", "deepCoadd_calexp", "goodSeeingDiff_differenceExp"). - data_id : `dict` or `DataCoordinate` + data_id : `dict` or `lsst.daf.butler.DataCoordinate` Data ID used to find the dataset (e.g. {"visit": 12345, "detector": 42}). - collections : `Sequence[str]` + collections : `collections.abc.Sequence[str]` Collections to search for the dataset. Returns ------- - uri : `ResourcePath` + uri : `lsst.resources.ResourcePath` Full path to the extracted PSF image file. """ psf_result = self.extract_search(ra, dec, dataset_type_name, data_id, collections) @@ -193,7 +193,7 @@ def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: ---------- ra, dec : `float` RA/Dec of the point where the PSF should be evaluated (in degrees). - ref : `DatasetRef` + ref : `lsst.daf.butler.DatasetRef` Fully-resolved dataset reference. Returns @@ -294,9 +294,9 @@ def extract_search( dataset_type_name : `str` Dataset type name of the image (e.g. "calexp", "deepCoadd_calexp", "goodSeeingDiff_differenceExp"). - data_id : `DataId` + data_id : `lsst.daf.butler.DataId` Data ID mapping used to locate the dataset. - collections : `Sequence[str]` + collections : `collections.abc.Sequence[str]` Collections to search for the dataset. Returns @@ -326,7 +326,7 @@ def write_fits(self, psf_result: PsfExtraction) -> ResourcePath: Returns ------- - uri : `ResourcePath` + uri : `lsst.resources.ResourcePath` Full path to the extracted PSF file. """ output_uuid = uuid4() From ca408b050dec4c8590040f5ede8f4fd3da355bd0 Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Fri, 20 Dec 2024 00:07:24 -0500 Subject: [PATCH 6/9] Enable caching for loaded skymaps --- python/lsst/psf/service/backend/projection_finders.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/lsst/psf/service/backend/projection_finders.py b/python/lsst/psf/service/backend/projection_finders.py index daea78a..35da38e 100644 --- a/python/lsst/psf/service/backend/projection_finders.py +++ b/python/lsst/psf/service/backend/projection_finders.py @@ -208,6 +208,8 @@ def find_projection(self, ref: DatasetRef, butler: Butler) -> tuple[SkyWcs, Box2 skymap = butler.get( self._dataset_type_name, skymap=ref.dataId["skymap"], collections=self._collections ) + # Populate the cache with the skymap to avoid reloading it. + self._cache[cast(str, ref.dataId["skymap"])] = skymap tractInfo = skymap[ref.dataId["tract"]] if "patch" in ref.dataId.dimensions: patchInfo = tractInfo[ref.dataId["patch"]] From a99cd9ba929fd81126f54c492de4d3210b7e43ce Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Fri, 20 Dec 2024 01:21:41 -0500 Subject: [PATCH 7/9] Refactor to skip full image reads and remove redundant component args --- .../service/backend/_psf_service_backend.py | 30 +++++++------------ tests/test_psfServiceBackend.py | 2 +- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/python/lsst/psf/service/backend/_psf_service_backend.py b/python/lsst/psf/service/backend/_psf_service_backend.py index 5be44e5..56f96c2 100644 --- a/python/lsst/psf/service/backend/_psf_service_backend.py +++ b/python/lsst/psf/service/backend/_psf_service_backend.py @@ -68,9 +68,9 @@ class PsfServiceBackend: - `deepCoadd_calexp` (calibrated coadd exposures), - `deepDiff_differenceExp` (difference image produced in AP pipeline). - Any dataset type that has a `getPsf()` method should be supported. Refer to - the LSST Data Management code on GitHub (https://github.com/lsst) for more - information on dataset types. + Any dataset type with a `getPsf()` method or a `psf` attribute should be + supported. Refer to https://github.com/lsst for more information on dataset + types. Parameters ---------- @@ -131,8 +131,6 @@ def process_uuid( ra: float, dec: float, uuid: UUID, - *, - component: str | None = None, ) -> ResourcePath: """Retrieve and write a PSF image from a dataset identified by its UUID. @@ -143,15 +141,13 @@ def process_uuid( RA/Dec of the point where the PSF should be evaluated (in degrees). uuid : `uuid.UUID` Unique ID of the dataset (e.g., a `calexp`). - component : `str`, optional - If not None, read this component instead of the composite dataset. Returns ------- uri : `lsst.resources.ResourcePath` Full path to the extracted PSF image file. """ - psf_result = self.extract_uuid(ra, dec, uuid, component=component) + psf_result = self.extract_uuid(ra, dec, uuid) return self.write_fits(psf_result) def process_search( @@ -215,15 +211,13 @@ def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: point_sky = geom.SpherePoint(geom.Angle(ra, geom.degrees), geom.Angle(dec, geom.degrees)) point_pixel = wcs.skyToPixel(point_sky) - # Get the image from the butler and extract the PSF model. - image = self.butler.get(ref) + # Get the ".psf" component of `ref` early to eliminate the need to read + # the full image just to access the PSF. + ref = ref.makeComponentRef("psf") - if not hasattr(image, "getPsf"): - raise ValueError( - f"The dataset {ref.datasetType.name} with ID {ref.id} does not have a `getPSF()` method" - ) + # Get the PSF model from the butler directly. + psf = self.butler.get(ref) - psf = image.getPsf() if psf is None: raise ValueError(f"No PSF found in dataset {ref.datasetType.name} with ID {ref.id}.") @@ -250,7 +244,7 @@ def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: ) def extract_uuid( - self, ra: float, dec: float, uuid: UUID, *, component: str | None = None + self, ra: float, dec: float, uuid: UUID ) -> PsfExtraction: """Extract a PSF image from a dataset identified by its UUID. @@ -260,8 +254,6 @@ def extract_uuid( RA/Dec (deg) of the PSF evaluation point. uuid : `UUID` Unique dataset identifier. - component : `str`, optional - If not None, read this component of the dataset. Returns ------- @@ -276,8 +268,6 @@ def extract_uuid( ref = self.butler.get_dataset(uuid) if ref is None: raise LookupError(f"No dataset found with UUID {uuid}.") - if component is not None: - ref = ref.makeComponentRef(component) return self.extract_ref(ra, dec, ref) diff --git a/tests/test_psfServiceBackend.py b/tests/test_psfServiceBackend.py index 5dd38cd..b942cdf 100644 --- a/tests/test_psfServiceBackend.py +++ b/tests/test_psfServiceBackend.py @@ -32,7 +32,7 @@ class TestPsfServiceBackend(lsst.utils.tests.TestCase): @classmethod def setUpClass(cls): try: - # Note: we are taking advantage of the exiting + # Note: we are taking advantage of the existing # `testdata_image_cutouts` here, as it contains the necessary # data to extract a PSF from and test the backend. cls.data_dir = lsst.utils.getPackageDir("testdata_image_cutouts") From 082463c456f34f18e5f228073675919895d3b952 Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Fri, 20 Dec 2024 01:48:03 -0500 Subject: [PATCH 8/9] Add option to choose between computeImage and computeKernelImage results --- .../service/backend/_psf_service_backend.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/python/lsst/psf/service/backend/_psf_service_backend.py b/python/lsst/psf/service/backend/_psf_service_backend.py index 56f96c2..10c0c35 100644 --- a/python/lsst/psf/service/backend/_psf_service_backend.py +++ b/python/lsst/psf/service/backend/_psf_service_backend.py @@ -24,15 +24,16 @@ __all__ = ("PsfServiceBackend", "PsfExtraction") import dataclasses -from uuid import UUID, uuid4 from collections.abc import Sequence +from uuid import UUID, uuid4 +import lsst.geom as geom from lsst.afw.image import ImageD from lsst.daf.base import PropertyList from lsst.daf.butler import Butler, DataId, DatasetRef from lsst.resources import ResourcePath, ResourcePathExpression + from .projection_finders import ProjectionFinder -import lsst.geom as geom @dataclasses.dataclass @@ -182,7 +183,9 @@ def process_search( psf_result = self.extract_search(ra, dec, dataset_type_name, data_id, collections) return self.write_fits(psf_result) - def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: + def extract_ref( + self, ra: float, dec: float, ref: DatasetRef, compute_kernel_image: bool = True + ) -> PsfExtraction: """Extract a PSF image from a fully-resolved `DatasetRef`. Parameters @@ -191,6 +194,9 @@ def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: RA/Dec of the point where the PSF should be evaluated (in degrees). ref : `lsst.daf.butler.DatasetRef` Fully-resolved dataset reference. + compute_kernel_image : `bool`, optional + If `True`, computes the PSF image using `computeKernelImage`; + otherwise uses `computeImage`. See Notes for more information. Returns ------- @@ -202,6 +208,14 @@ def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: ValueError If `ref.id` is not resolved or if the dataset does not contain a PSF. + + Notes + ----- + `computeKernelImage` centers the PSF at the middle of the central + pixel, thereby using a coordinate system with the PSF center at (0, 0). + `computeImage` intentionally shifts the PSF model to the specified + sub-pixel position, aligning it with the same coordinate system as the + pixelized image. """ if ref.id is None: raise ValueError(f"A resolved DatasetRef with a valid ID is required; got {ref}.") @@ -221,8 +235,11 @@ def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: if psf is None: raise ValueError(f"No PSF found in dataset {ref.datasetType.name} with ID {ref.id}.") - # Compute the PSF kernel image at the given point using the PSF model. - psf_image = psf.computeKernelImage(point_pixel) + # Compute the PSF image at the given point using the PSF model. + if compute_kernel_image: + psf_image = psf.computeKernelImage(point_pixel) + else: + psf_image = psf.computeImage(point_pixel) # Create FITS metadata. metadata = PropertyList() @@ -243,9 +260,7 @@ def extract_ref(self, ra: float, dec: float, ref: DatasetRef) -> PsfExtraction: origin_ref=ref, ) - def extract_uuid( - self, ra: float, dec: float, uuid: UUID - ) -> PsfExtraction: + def extract_uuid(self, ra: float, dec: float, uuid: UUID) -> PsfExtraction: """Extract a PSF image from a dataset identified by its UUID. Parameters From c09849cd8965d4619f5c9f2226af897dd5c23e7c Mon Sep 17 00:00:00 2001 From: Erfan Nourbakhsh Date: Fri, 20 Dec 2024 15:33:10 -0500 Subject: [PATCH 9/9] Clarify dataset ref and pass compute_kernel_image through all extract/process methods --- .../service/backend/_psf_service_backend.py | 57 +++++++++++++++---- 1 file changed, 45 insertions(+), 12 deletions(-) diff --git a/python/lsst/psf/service/backend/_psf_service_backend.py b/python/lsst/psf/service/backend/_psf_service_backend.py index 10c0c35..6e94147 100644 --- a/python/lsst/psf/service/backend/_psf_service_backend.py +++ b/python/lsst/psf/service/backend/_psf_service_backend.py @@ -107,7 +107,9 @@ def __init__( output_root: ResourcePath temporary_root: ResourcePath | None - def process_ref(self, ra: float, dec: float, ref: DatasetRef) -> ResourcePath: + def process_ref( + self, ra: float, dec: float, ref: DatasetRef, compute_kernel_image: bool = True + ) -> ResourcePath: """Retrieve and write a PSF image from a fully-resolved `DatasetRef`. Parameters @@ -116,15 +118,19 @@ def process_ref(self, ra: float, dec: float, ref: DatasetRef) -> ResourcePath: Right Ascension and Declination of the point (in degrees) where the PSF should be evaluated. ref : `lsst.daf.butler.DatasetRef` - Fully-resolved reference to a dataset (e.g., `calexp`, - `deepCoadd_calexp`, `goodSeeingDiff_differenceExp`). + Fully-resolved reference to a dataset containing the PSF (e.g., + `calexp`, `deepCoadd_calexp`, `goodSeeingDiff_differenceExp`). + This does not directly point to the PSF component of the dataset. + compute_kernel_image : `bool`, optional + If `True`, computes the PSF image using `computeKernelImage`; + otherwise uses `computeImage`. Returns ------- uri : `lsst.resources.ResourcePath` Full path to the extracted PSF image file. """ - psf_result = self.extract_ref(ra, dec, ref) + psf_result = self.extract_ref(ra, dec, ref, compute_kernel_image) return self.write_fits(psf_result) def process_uuid( @@ -132,6 +138,7 @@ def process_uuid( ra: float, dec: float, uuid: UUID, + compute_kernel_image: bool = True, ) -> ResourcePath: """Retrieve and write a PSF image from a dataset identified by its UUID. @@ -142,13 +149,16 @@ def process_uuid( RA/Dec of the point where the PSF should be evaluated (in degrees). uuid : `uuid.UUID` Unique ID of the dataset (e.g., a `calexp`). + compute_kernel_image : `bool`, optional + If `True`, computes the PSF image using `computeKernelImage`; + otherwise uses `computeImage`. Returns ------- uri : `lsst.resources.ResourcePath` Full path to the extracted PSF image file. """ - psf_result = self.extract_uuid(ra, dec, uuid) + psf_result = self.extract_uuid(ra, dec, uuid, compute_kernel_image) return self.write_fits(psf_result) def process_search( @@ -158,6 +168,7 @@ def process_search( dataset_type_name: str, data_id: DataId, collections: Sequence[str], + compute_kernel_image: bool = True, ) -> ResourcePath: """Retrieve and write a PSF image from a dataset identified by (dataset type, data ID, collection). @@ -174,13 +185,18 @@ def process_search( "detector": 42}). collections : `collections.abc.Sequence[str]` Collections to search for the dataset. + compute_kernel_image : `bool`, optional + If `True`, computes the PSF image using `computeKernelImage`; + otherwise uses `computeImage`. Returns ------- uri : `lsst.resources.ResourcePath` Full path to the extracted PSF image file. """ - psf_result = self.extract_search(ra, dec, dataset_type_name, data_id, collections) + psf_result = self.extract_search( + ra, dec, dataset_type_name, data_id, collections, compute_kernel_image + ) return self.write_fits(psf_result) def extract_ref( @@ -191,9 +207,12 @@ def extract_ref( Parameters ---------- ra, dec : `float` - RA/Dec of the point where the PSF should be evaluated (in degrees). + Right Ascension and Declination of the point (in degrees) where the + PSF should be evaluated. ref : `lsst.daf.butler.DatasetRef` - Fully-resolved dataset reference. + Fully-resolved reference to a dataset containing the PSF (e.g., + `calexp`, `deepCoadd_calexp`, `goodSeeingDiff_differenceExp`). + This does not directly point to the PSF component of the dataset. compute_kernel_image : `bool`, optional If `True`, computes the PSF image using `computeKernelImage`; otherwise uses `computeImage`. See Notes for more information. @@ -260,7 +279,9 @@ def extract_ref( origin_ref=ref, ) - def extract_uuid(self, ra: float, dec: float, uuid: UUID) -> PsfExtraction: + def extract_uuid( + self, ra: float, dec: float, uuid: UUID, compute_kernel_image: bool = True + ) -> PsfExtraction: """Extract a PSF image from a dataset identified by its UUID. Parameters @@ -269,6 +290,9 @@ def extract_uuid(self, ra: float, dec: float, uuid: UUID) -> PsfExtraction: RA/Dec (deg) of the PSF evaluation point. uuid : `UUID` Unique dataset identifier. + compute_kernel_image : `bool`, optional + If `True`, computes the PSF image using `computeKernelImage`; + otherwise uses `computeImage`. Returns ------- @@ -284,10 +308,16 @@ def extract_uuid(self, ra: float, dec: float, uuid: UUID) -> PsfExtraction: if ref is None: raise LookupError(f"No dataset found with UUID {uuid}.") - return self.extract_ref(ra, dec, ref) + return self.extract_ref(ra, dec, ref, compute_kernel_image) def extract_search( - self, ra: float, dec: float, dataset_type_name: str, data_id: DataId, collections: Sequence[str] + self, + ra: float, + dec: float, + dataset_type_name: str, + data_id: DataId, + collections: Sequence[str], + compute_kernel_image: bool = True, ) -> PsfExtraction: """Extract a PSF image from a dataset identified by (dataset type, data ID, collection). @@ -303,6 +333,9 @@ def extract_search( Data ID mapping used to locate the dataset. collections : `collections.abc.Sequence[str]` Collections to search for the dataset. + compute_kernel_image : `bool`, optional + If `True`, computes the PSF image using `computeKernelImage`; + otherwise uses `computeImage`. Returns ------- @@ -319,7 +352,7 @@ def extract_search( raise LookupError( f"No {dataset_type_name} dataset found with data ID {data_id} in {collections}." ) - return self.extract_ref(ra, dec, ref) + return self.extract_ref(ra, dec, ref, compute_kernel_image) def write_fits(self, psf_result: PsfExtraction) -> ResourcePath: """Write a `PsfExtraction` to a FITS file in `output_root`.