diff --git a/poetry.lock b/poetry.lock index 777b2fe..634180d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -206,33 +206,33 @@ test = ["astroid (>=1,<2)", "astroid (>=2,<4)", "pytest"] [[package]] name = "black" -version = "24.4.0" +version = "24.4.1" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-24.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6ad001a9ddd9b8dfd1b434d566be39b1cd502802c8d38bbb1ba612afda2ef436"}, - {file = "black-24.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3a3a092b8b756c643fe45f4624dbd5a389f770a4ac294cf4d0fce6af86addaf"}, - {file = "black-24.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dae79397f367ac8d7adb6c779813328f6d690943f64b32983e896bcccd18cbad"}, - {file = "black-24.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:71d998b73c957444fb7c52096c3843875f4b6b47a54972598741fe9a7f737fcb"}, - {file = "black-24.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8e5537f456a22cf5cfcb2707803431d2feeb82ab3748ade280d6ccd0b40ed2e8"}, - {file = "black-24.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64e60a7edd71fd542a10a9643bf369bfd2644de95ec71e86790b063aa02ff745"}, - {file = "black-24.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cd5b4f76056cecce3e69b0d4c228326d2595f506797f40b9233424e2524c070"}, - {file = "black-24.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:64578cf99b6b46a6301bc28bdb89f9d6f9b592b1c5837818a177c98525dbe397"}, - {file = "black-24.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f95cece33329dc4aa3b0e1a771c41075812e46cf3d6e3f1dfe3d91ff09826ed2"}, - {file = "black-24.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4396ca365a4310beef84d446ca5016f671b10f07abdba3e4e4304218d2c71d33"}, - {file = "black-24.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44d99dfdf37a2a00a6f7a8dcbd19edf361d056ee51093b2445de7ca09adac965"}, - {file = "black-24.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:21f9407063ec71c5580b8ad975653c66508d6a9f57bd008bb8691d273705adcd"}, - {file = "black-24.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:652e55bb722ca026299eb74e53880ee2315b181dfdd44dca98e43448620ddec1"}, - {file = "black-24.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7f2966b9b2b3b7104fca9d75b2ee856fe3fdd7ed9e47c753a4bb1a675f2caab8"}, - {file = "black-24.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bb9ca06e556a09f7f7177bc7cb604e5ed2d2df1e9119e4f7d2f1f7071c32e5d"}, - {file = "black-24.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4e71cdebdc8efeb6deaf5f2deb28325f8614d48426bed118ecc2dcaefb9ebf3"}, - {file = "black-24.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6644f97a7ef6f401a150cca551a1ff97e03c25d8519ee0bbc9b0058772882665"}, - {file = "black-24.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:75a2d0b4f5eb81f7eebc31f788f9830a6ce10a68c91fbe0fade34fff7a2836e6"}, - {file = "black-24.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb949f56a63c5e134dfdca12091e98ffb5fd446293ebae123d10fc1abad00b9e"}, - {file = "black-24.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:7852b05d02b5b9a8c893ab95863ef8986e4dda29af80bbbda94d7aee1abf8702"}, - {file = "black-24.4.0-py3-none-any.whl", hash = "sha256:74eb9b5420e26b42c00a3ff470dc0cd144b80a766128b1771d07643165e08d0e"}, - {file = "black-24.4.0.tar.gz", hash = "sha256:f07b69fda20578367eaebbd670ff8fc653ab181e1ff95d84497f9fa20e7d0641"}, + {file = "black-24.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f7749fd0d97ff9415975a1432fac7df89bf13c3833cea079e55fa004d5f28c0"}, + {file = "black-24.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:859f3cc5d2051adadf8fd504a01e02b0fd866d7549fff54bc9202d524d2e8bd7"}, + {file = "black-24.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59271c9c29dfa97f7fda51f56c7809b3f78e72fd8d2205189bbd23022a0618b6"}, + {file = "black-24.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:5ed9c34cba223149b5a0144951a0f33d65507cf82c5449cb3c35fe4b515fea9a"}, + {file = "black-24.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9dae3ae59d6f2dc93700fd5034a3115434686e66fd6e63d4dcaa48d19880f2b0"}, + {file = "black-24.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5f8698974a81af83283eb47644f2711b5261138d6d9180c863fce673cbe04b13"}, + {file = "black-24.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f404b6e77043b23d0321fb7772522b876b6de737ad3cb97d6b156638d68ce81"}, + {file = "black-24.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:c94e52b766477bdcd010b872ba0714d5458536dc9d0734eff6583ba7266ffd89"}, + {file = "black-24.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:962d9e953872cdb83b97bb737ad47244ce2938054dc946685a4cad98520dab38"}, + {file = "black-24.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b1d8e3b2486b7dd522b1ab2ba1ec4907f0aa8f5e10a33c4271fb331d1d10b70c"}, + {file = "black-24.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed77e214b785148f57e43ca425b6e0850165144aa727d66ac604e56a70bb7825"}, + {file = "black-24.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:4ef4764437d7eba8386689cd06e1fb5341ee0ae2e9e22582b21178782de7ed94"}, + {file = "black-24.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:92b183f8eef5baf7b20a513abcf982ad616f544f593f6688bb2850d2982911f1"}, + {file = "black-24.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:945abd7b3572add997757c94295bb3e73c6ffaf3366b1f26cb2356a4bffd1dc3"}, + {file = "black-24.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db5154b9e5b478031371d8bc41ff37b33855fa223a6cfba456c9b73fb96f77d4"}, + {file = "black-24.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:afc84c33c1a9aaf3d73140cee776b4ddf73ff429ffe6b7c56dc1c9c10725856d"}, + {file = "black-24.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0889f4eb8b3bdf8b189e41a71cf0dbb8141a98346cd1a2695dea5995d416e940"}, + {file = "black-24.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5bb0143f175db45a55227eefd63e90849d96c266330ba31719e9667d0d5ec3b9"}, + {file = "black-24.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:713a04a78e78f28ef7e8df7a16fe075670ea164860fcef3885e4f3dffc0184b3"}, + {file = "black-24.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:171959bc879637a8cdbc53dc3fddae2a83e151937a28cf605fd175ce61e0e94a"}, + {file = "black-24.4.1-py3-none-any.whl", hash = "sha256:ecbab810604fe02c70b3a08afd39beb599f7cc9afd13e81f5336014133b4fe35"}, + {file = "black-24.4.1.tar.gz", hash = "sha256:5241612dc8cad5b6fd47432b8bd04db80e07cfbc53bb69e9ae18985063bcb8dd"}, ] [package.dependencies] @@ -1751,6 +1751,7 @@ optional = false python-versions = ">=3.9" files = [ {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, + {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"}, {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, @@ -1771,6 +1772,7 @@ files = [ {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, + {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"}, {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, @@ -2532,6 +2534,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -2867,13 +2870,13 @@ docs = ["Sphinx (>=4.0.0)", "furo (>=2021.6.18-beta.36)", "myst-parser (>=0.15)" [[package]] name = "sdssdb" -version = "0.10.0" +version = "0.11.1" description = "SDSS product for database management" optional = false python-versions = ">=3.6" files = [ - {file = "sdssdb-0.10.0-py3-none-any.whl", hash = "sha256:51595f0fd046f0d0ce9c14575a5a365650bad3a9374281eadf974a4b58ce0c02"}, - {file = "sdssdb-0.10.0.tar.gz", hash = "sha256:26ba2c762f569a121c689cc6eff48cdc3ca098e2ddf50d488729025446bc1a4e"}, + {file = "sdssdb-0.11.1-py3-none-any.whl", hash = "sha256:acace0978b809a1527cd90b328a1a604ec1887e996aea04d543ad47feec3dc66"}, + {file = "sdssdb-0.11.1.tar.gz", hash = "sha256:8c8a54518f9cdaca91bc34ced22529b81f3373cc13e3ae5dec1c50ba653a5320"}, ] [package.dependencies] @@ -3211,4 +3214,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12,<4.0" -content-hash = "2d7f744b6ab334586229d0b25346e23c8d10083880d7ffbb658cc38a6d901dd9" +content-hash = "4514ad971c967a83a4b2917e9c6563c480baa18fbcf6f79ae7e702dbb78859b2" diff --git a/pyproject.toml b/pyproject.toml index 0b35dde..0a90375 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ adbc-driver-postgresql = "^0.9.0" polars = "^0.20.7" httpx = "^0.26.0" rich = "^13.7.0" -sdssdb = "^0.10.0" +sdssdb = "^0.11.1" pyarrow = "^15.0.0" click-option-group = "^0.5.6" astropy-healpix = "^1.0.2" @@ -76,8 +76,7 @@ asyncio_mode = "auto" branch = true include = ["src/too/*"] omit = [ - "*/__init__.py", - "src/too/validate.py" + "*/__init__.py" ] [tool.coverage.report] diff --git a/src/too/validate.py b/src/too/validate.py index d13422c..7f0c4b6 100644 --- a/src/too/validate.py +++ b/src/too/validate.py @@ -15,7 +15,6 @@ from typing import TYPE_CHECKING, Tuple import astropy.units as u -import numpy import numpy as np import polars from astropy.coordinates import SkyCoord @@ -37,7 +36,7 @@ from sdssdb import PeeweeDatabaseConnection -__all__ = ["validate_too_targets", "validate_bright_limits"] +__all__ = ["validate_too_targets", "add_bright_limits_columns"] # load enviornment variable with path to healpix maps @@ -47,7 +46,11 @@ fmagloss = Moffat2dInterp() -def check_assign_mag_limit(mag_metric_min, mag_metric_max, assign_mag): +def check_assign_mag_limit( + mag_metric_min, + mag_metric_max, + assign_mag, +): # pragma: no cover """ Checks the if magnitude of one assignment agrees with design mode for some instrument and carton class @@ -263,7 +266,7 @@ def fromdb(self, label: str): return -def magnitude_array(targets: polars.DataFrame) -> np.ndarray: +def magnitude_array(targets: polars.DataFrame) -> np.ndarray: # pragma: no cover """ create the magnitude array for the targets Parameters @@ -310,7 +313,7 @@ def calculate_offsets( modes: dict, offset_min_skybrightness: float = 0.0, observatory: str = "APO", -) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: # pragma: no cover """ Calculate the offsets for the targets @@ -395,7 +398,7 @@ def bn_validation( design_mode: str, modes: dict, observatory: str = "APO", -) -> np.ndarray: +) -> np.ndarray: # pragma: no cover """ Validate a ToO to see if it is too close to a bright neighbor. This functionm relies on the @@ -435,12 +438,12 @@ def bn_validation( # load the healpix indicies for the designmode bn_maps_boss_file = f"{BN_HEALPIX}/{design_mode}_boss_bn_healpix.parquet" if not os.path.exists(bn_maps_boss_file): - raise ValueError(f"File {bn_maps_boss_file} does not exist.") + raise FileNotFoundError(f"File {bn_maps_boss_file} does not exist.") bn_maps_boss = polars.scan_parquet(bn_maps_boss_file) bn_maps_apogee_file = f"{BN_HEALPIX}/{design_mode}_apogee_bn_healpix.parquet" if not os.path.exists(bn_maps_apogee_file): - raise ValueError(f"File {bn_maps_apogee_file} does not exist.") + raise FileNotFoundError(f"File {bn_maps_apogee_file} does not exist.") bn_maps_apogee = polars.scan_parquet(bn_maps_apogee_file) # create the correct nside healpix object @@ -617,7 +620,7 @@ def mag_lim_validation( def validate_too_targets( targets: polars.DataFrame, database: PeeweeDatabaseConnection, - drop_bright_targets: bool = False, + bright_limit_checks: bool = False, ): """Validates a list of ToO targets. @@ -630,6 +633,10 @@ def validate_too_targets( - The number of exposures is set and valid. - The ``active`` column is set. + If ``bright_limit_checks`` is ``True``, the function will also run the bright + neighbour and bright limit checks and will fail if any of the targets do not pass + the checks. + """ n_targets = targets.height @@ -694,25 +701,32 @@ def validate_too_targets( if targets["can_offset"].is_null().any(): raise ValidationError("Null 'can_offset' column values found in ToO targets.") - # Validate magnitude limits and bright neighbours. - log.info("Running bright neighbour and magnitude limit checks.") - bn_targets = validate_bright_limits(targets, database) + # Check that the sky_brightness_mode value are valid. + valid_brightness_modes = ["bright", "dark"] + + targets_invalid_brightness_mode = targets.filter( + polars.col.sky_brightness_mode.is_in(valid_brightness_modes).not_() + ) + if len(targets_invalid_brightness_mode) > 0: + raise ValidationError("Invalid sky_brightness_mode values found.") + + if bright_limit_checks: + # Validate magnitude limits and bright neighbours. + log.info("Running bright neighbour and magnitude limit checks.") + bn_targets = add_bright_limits_columns(targets, database) + + # Check if any targets failed the bright neighbour or magnitude limit checks. + bn_columns = bn_targets.select(polars.selectors.starts_with("bn_")) + bn_invalid = bn_targets.filter(~bn_columns.fold(lambda x, y: x & y)) - # Require both checks to pass. - bn_invalid = bn_targets.filter(~polars.col.bn_valid | ~polars.col.mag_lim_valid) + mag_lim_columns = bn_targets.select(polars.selectors.starts_with("mag_lim_")) + mag_lim_invalid = bn_targets.filter(~mag_lim_columns.fold(lambda x, y: x & y)) - if len(bn_invalid) > 0: - if not drop_bright_targets: + if len(bn_invalid) > 0 or len(mag_lim_invalid) > 0: raise ValidationError( f"{len(bn_invalid)} targets failed bright neighbour or " "magnitude limit checks." ) - else: - log.warning( - f"{len(bn_invalid)} targets failed bright neighbour or " - "magnitude limit checks and will be rejected." - ) - targets = targets.filter(~polars.col.too_id.is_in(bn_invalid["too_id"])) # Fill some optional columns. targets = targets.with_columns( @@ -729,11 +743,28 @@ def validate_too_targets( return targets -def validate_bright_limits( +def add_bright_limits_columns( targets: polars.DataFrame, database: PeeweeDatabaseConnection, ): - """Runs the Mugatu-like validation for bright targets.""" + """Runs the Mugatu-like validation for bright targets. + + Parameters + ---------- + targets + The ToO targets to validate. + database + The database connection to use to query the design modes. + + Returns + ------- + data_frame + A data frame with the same rows as the input ``targets`` with additional + columns that indicate if a target passes the bright neighbour and magnitude + limit checks. The order of the returned data frame is sorted by ``too_id`` + and may be different as that of the input one. + + """ targets = targets.clone() @@ -743,17 +774,10 @@ def validate_bright_limits( # Check that the sky_brightness_mode value are valid. valid_brightness_modes = ["bright", "dark"] - targets_invalid_brightness_mode = targets.filter( - polars.col.sky_brightness_mode.is_in(valid_brightness_modes).not_() - ) - if len(targets_invalid_brightness_mode) > 0: - raise ValidationError("Invalid sky_brightness_mode values found.") - # Get a list of design modes. design_modes: dict[str, DesignMode] = allDesignModes(database) - # List of validated targets for each brightness mode. - targets_bmode_validated: list[polars.DataFrame] = [] + processed: list[polars.DataFrame] = [] # First we split the targets by observatory and sky_brightness_mode (bright or # dark). Then we run the check for all the design modes that match that brightness @@ -768,48 +792,54 @@ def validate_bright_limits( design_modes_bmode = [dm for dm in design_modes if dm.startswith(bmode)] - valid_bn: list[numpy.ndarray] = [] - valid_mag_lim: list[numpy.ndarray] = [] - for dmb in design_modes_bmode: - log.debug(f"Validating bright neighbours for design mode: {dmb}") + log.debug( + f"Validating bright neighbours for observatory {observatory} " + f"and design mode {dmb}" + ) + try: with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ErfaWarning) - valid_bn.append( - bn_validation( - targets_bmode, - dmb, - design_modes, - observatory=observatory, - ) + bn_mask = bn_validation( + targets_bmode, + dmb, + design_modes, + observatory=observatory, ) - valid_mag_lim.append( - mag_lim_validation( - targets_bmode, - dmb, - design_modes, - observatory=observatory, - ) - ) + except FileNotFoundError: + # If there is not a BN file for this design mode, we just skip it + # and assume all targets pass the check. + bn_mask = np.ones(len(targets_bmode), dtype=bool) - except Exception as err: - log.warning( - f"Error validating targets for design mode {dmb!r}: {err}" - ) + except Exception: + raise - if len(valid_bn) == 0 or len(valid_mag_lim) == 0: - raise ValidationError("Error validating magnitude limits.") + mag_lim_mask = mag_lim_validation( + targets_bmode, + dmb, + design_modes, + observatory=observatory, + ) - valid_bn_1d = numpy.all(numpy.stack(valid_bn), axis=0) - valid_mag_lim_1d = numpy.all(numpy.stack(valid_mag_lim), axis=0) + targets_bmode = targets_bmode.with_columns( + **{ + f"bn_{dmb}_valid": polars.Series(bn_mask), + f"mag_lim_{dmb}_valid": polars.Series(mag_lim_mask), + } + ) - targets_bmode = targets_bmode.with_columns( - bn_valid=polars.Series(valid_bn_1d), - mag_lim_valid=polars.Series(valid_mag_lim_1d), - ) - targets_bmode_validated.append(targets_bmode) + processed.append(targets_bmode) + + # Concatenate all the processed data frames. For targets that do not have a + # corresponding design mode, the columns will be nulls. We fill those with False + # as in principle we should not be able to observe the targets in those modes. + processed_df = polars.concat(processed, how="diagonal") + processed_df = processed_df.with_columns( + polars.selectors.starts_with("bn_").fill_null(False), + polars.selectors.starts_with("mag_lim_").fill_null(False), + ) - return polars.concat(targets_bmode_validated) + return processed_df.sort("too_id") diff --git a/tests/conftest.py b/tests/conftest.py index 78ea4be..d883f49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,7 +83,6 @@ def mock_bn_mag_lim(bn_targets, *args, **kargs): return numpy.ones(len(bn_targets), dtype=bool) mocker.patch.object(too.validate, "bn_validation", side_effect=mock_bn_mag_lim) - mocker.patch.object(too.validate, "mag_lim_validation", side_effect=mock_bn_mag_lim) @pytest.fixture() diff --git a/tests/test_validate.py b/tests/test_validate.py index 9326569..b0fa686 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -10,11 +10,12 @@ from typing import TYPE_CHECKING +import numpy import pytest import too.validate from too.exceptions import ValidationError -from too.validate import validate_bright_limits, validate_too_targets +from too.validate import add_bright_limits_columns, validate_too_targets if TYPE_CHECKING: @@ -89,51 +90,91 @@ def test_validate_too_target_fails( validate_too_targets(too_mock_test, database) -def test_validate_bright_limits_fails( +def test_add_bright_limits_columns_fails( too_mock: polars.DataFrame, database: PeeweeDatabaseConnection, mocker: pytest_mock.MockerFixture, - caplog: pytest.LogCaptureFixture, ): mocker.patch.object(too.validate, "bn_validation", side_effect=RuntimeError) - with pytest.raises(ValidationError): - validate_bright_limits(too_mock, database) - - assert "Error validating targets for design mode" in caplog.record_tuples[-1][2] + with pytest.raises(RuntimeError): + add_bright_limits_columns(too_mock, database) -@pytest.mark.parametrize("drop_bright_targets", [True, False]) def test_validate_too_targets_bn_invalid( too_mock: polars.DataFrame, database: PeeweeDatabaseConnection, mocker: pytest_mock.MockerFixture, - drop_bright_targets: bool, - caplog: pytest.LogCaptureFixture, ): too_mock_bn = too_mock.with_columns( - bn_valid=polars.lit(True, dtype=polars.Boolean), - mag_lim_valid=polars.lit(True, dtype=polars.Boolean), + bn_dark_monit_valid=polars.lit(True, dtype=polars.Boolean), + mag_lim_dark_monit_valid=polars.lit(True, dtype=polars.Boolean), ) too_mock_bn[0, -1] = False mocker.patch.object( too.validate, - "validate_bright_limits", + "add_bright_limits_columns", return_value=too_mock_bn, ) - if not drop_bright_targets: - with pytest.raises(ValidationError): - validate_too_targets( - too_mock, - database, - drop_bright_targets=drop_bright_targets, - ) - else: + with pytest.raises(ValidationError): validate_too_targets( too_mock, database, - drop_bright_targets=drop_bright_targets, + bright_limit_checks=True, ) - assert "1 targets failed bright neighbour" in caplog.record_tuples[-1][2] + + +def test_add_bright_limits_columns( + too_mock: polars.DataFrame, + database: PeeweeDatabaseConnection, + mocker: pytest_mock.MockerFixture, +): + # Simulate that bn_validation returns all False + mocker.patch.object( + too.validate, + "bn_validation", + side_effect=lambda targets, *_, **__: numpy.zeros(len(targets)).astype(bool), + ) + + df = add_bright_limits_columns(too_mock[0:1000], database) + + assert isinstance(df, polars.DataFrame) + assert "bn_dark_monit_valid" in df.columns + assert not df[0, "bn_dark_monit_valid"] + assert not df["mag_lim_dark_monit_valid"].all() + + +def test_add_bright_limits_columns_filenotfound( + too_mock: polars.DataFrame, + database: PeeweeDatabaseConnection, + mocker: pytest_mock.MockerFixture, +): + mocker.patch.object( + too.validate, + "bn_validation", + side_effect=FileNotFoundError, + ) + + df = add_bright_limits_columns(too_mock[0:1000], database) + + assert isinstance(df, polars.DataFrame) + assert "bn_dark_monit_valid" in df.columns + + assert df["bn_dark_monit_valid"].any() + + +def test_add_bright_limits_columns_raises( + too_mock: polars.DataFrame, + database: PeeweeDatabaseConnection, + mocker: pytest_mock.MockerFixture, +): + mocker.patch.object( + too.validate, + "bn_validation", + side_effect=ValueError, + ) + + with pytest.raises(ValueError): + add_bright_limits_columns(too_mock[0:1000], database)