From ed63002f63b30027bdc4ab77da7e98ea13b2b15b Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Sun, 19 May 2024 17:30:24 -0700 Subject: [PATCH 1/5] Fix some minor problems. --- python/lsst/consdb/hinfo.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index 7b3015fa..f19b7b6a 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -39,7 +39,7 @@ def tai_mean(start: str, end: str) -> datetime: return (s + (e - s) / 2).datetime -def mean(*iterable: float) -> Any: +def mean(*iterable: float) -> float: return sum(iterable) / len(iterable) @@ -100,12 +100,11 @@ def logical_or(*bools: int | str | None) -> bool: "dome_azimuth": "DOMEAZ", "shut_lower": "SHUTLOWR", "shut_upper": "SHUTUPPR", - # "temp_set": "TEMP_SET", "simulated": ( logical_or, "SIMULATE ATMCS", "SIMULATE ATHEXAPOD", - "SIMULAT ATPNEUMATICS", + "SIMULATE ATPNEUMATICS", "SIMULATE ATDOME", "SIMULATE ATSPECTROGRAPH", ), From 6586bb32a9dfb50754a6ca1373d2c8b8bffc7f7e Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Sun, 19 May 2024 17:31:06 -0700 Subject: [PATCH 2/5] Add focal plane and detector regions. --- python/lsst/consdb/hinfo.py | 123 +++++++++++++++++++++++++++++++----- 1 file changed, 106 insertions(+), 17 deletions(-) diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index f19b7b6a..00c4d8ed 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -13,8 +13,11 @@ import httpx import kafkit.registry import kafkit.registry.httpx +import lsst.geom +import lsst.obs.lsst import yaml from astro_metadata_translator import ObservationInfo +from lsst.obs.lsst.rawFormatter import LsstCamRawFormatter from lsst.resources import ResourcePath from sqlalchemy import MetaData, Table from sqlalchemy.dialects.postgresql import insert @@ -47,6 +50,63 @@ def logical_or(*bools: int | str | None) -> bool: return any([b == 1 or b == "1" for b in bools]) +def region(ra: float, dec: float, rotpa: float, points: list[tuple[str, int, int]]) -> str: + global camera, formatter + region = "Polygon ICRS" + for detector, offset_x, offset_y in points: + skywcs = formatter.makeRawSkyWcsFromBoresight( + lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees), + rotpa*lsst.geom.degrees, + camera[detector], + ) + bbox = camera[detector].getBBox() + x = bbox.getMinX() + bbox.getWidth() * offset_x + y = bbox.getMinY() + bbox.getHeight() * offset_y + point = skywcs.pixelToSky(x, y) + region += f" {point.getLongitude().asDegrees():.6f} {point.getLatitude.asDegrees():.6f}" + return region + + +def ccdexposure_id(exposure_id: int, detector: int) -> int: + global translator + return translator.compute_detector_exposure_id(exposure_id, detector) + + +def ccd_region(imgtype: str, ra: float, dec: float, rotpa: float, ccdname: str) -> str | None: + if imgtype != "OBJECT": + return None + return region(ra, dec, rotpa, [ + (ccdname, 0, 0), (ccdname, 1, 0), (ccdname, 1, 1), (ccdname, 0, 1), + ]) + + +def fp_region(imgtype: str, ra: float, dec: float, rotpa: float) -> str | None: + global instrument + if imgtype != "OBJECT": + return None + if instrument == "LATISS": + corners = [ + ("RXX_S00", 0, 0), ("RXX_S00", 1, 0), ("RXX_S00", 1, 1), ("RXX_S00", 0, 1), + ] + elif instrument == "LSSTComCam" or instrument == "LSSTComCamSim": + corners = [ + ("R22_S00", 0, 0), ("R22_S02", 1, 0), ("R22_S20", 0, 1), ("R22_S22", 1, 1), + ] + elif instrument == "LSSTCam": + corners = [ + ("R01_S00", 0, 0), ("R03_S02", 1, 0), ("R03_S02", 1, 1), ("R04_SG1", 1, 0), + ("R04_SG1", 1, 1), ("R04_SG0", 1, 0), ("R04_SG0", 1, 1), ("R14_S02", 1, 0), + ("R34_S22", 1, 1), ("R34_S22", 0, 1), ("R44_SG1", 1, 1), ("R44_SG1", 0, 1), + ("R44_SG0", 1, 1), ("R44_SG0", 0, 1), ("R43_S22", 1, 1), ("R41_S20", 0, 1), + ("R41_S20", 0, 0), ("R40_SG1", 0, 1), ("R40_SG1", 0, 0), ("R40_SG0", 0, 1), + ("R40_SG0", 0, 0), ("R30_S20", 0, 1), ("R10_S00", 0, 0), ("R10_S00", 1, 0), + ("R00_SG1", 0, 0), ("R00_SG1", 1, 0), ("R00_SG0", 0, 0), ("R00_SG0", 1, 0), + ] + else: + return None + return region(ra, dec, rotpa, corners) + + ################################# # Header Mapping Configurations # ################################# @@ -92,11 +152,12 @@ def logical_or(*bools: int | str | None) -> bool: "wind_speed": "WINDSPD", "wind_dir": "WINDDIR", "dimm_seeing": "SEEING", + "focus_z": "FOCUSZ", + "s_region": (fp_region, "IMGTYPE", "RA", "DEC", "ROTPA"), } # Instrument-specific mapping to column name from Header Service keyword LATISS_MAPPING: dict[str, str | Sequence] = { - "focus_z": "FOCUSZ", "dome_azimuth": "DOMEAZ", "shut_lower": "SHUTLOWR", "shut_upper": "SHUTUPPR", @@ -110,13 +171,26 @@ def logical_or(*bools: int | str | None) -> bool: ), } -LSSTCOMCAM_MAPPING: dict[str, str | Sequence] = {} -LSSTCOMCAMSIM_MAPPING: dict[str, str | Sequence] = {} -LSSTCAM_MAPPING: dict[str, str | Sequence] = {} +LSSTCAM_MAPPING: dict[str, str | Sequence] = { + "simulated": ( + logical_or, + "SIMULATE MTMOUNT", + "SIMULATE MTM1M3", + "SIMULATE MTM2", + "SIMULATE CAMHEXAPOD", + "SIMULATE M2HEXAPOD", + "SIMULATE MTROTATOR", + "SIMULATE MTDOME", + "SIMULATE MTDOMETRAJECTORY", + ), +} -# LATISS_DETECTOR_MAPPING = { -# "ccdtemp": "CCDTEMP", -# } +DETECTOR_MAPPING = { + "ccdexposure_id": (ccdexposure_id, "exposure_id", "detector"), + "exposure_id": "exposure_id", + "detector": "detector", + "s_region": (ccd_region, "IMGTYPE", "RA", "DEC", "ROTPA", "_CCDNAME"), +} # Mapping to column name from ObservationInfo keyword OI_MAPPING = { @@ -182,7 +256,7 @@ def process_resource(resource: ResourcePath) -> None: resource: `ResourcePath` Path to the Header Service header resource. """ - global KW_MAPPING, OI_MAPPING, instrument_mapping, translator + global KW_MAPPING, OI_MAPPING, instrument_mapping, det_mapping, translator global engine, exposure_table exposure_rec = dict() @@ -208,12 +282,17 @@ def process_resource(resource: ResourcePath) -> None: with engine.begin() as conn: conn.execute(stmt) - # TODO: exposure_detector table processing - # det_info = dict() - # for header in content["R00S00_PRIMARY"]: - # det_info[header["keyword"]] = header["value"] - # for field, keyword in LATISS_DETECTOR_MAPPING.items(): - # det_exposure_rec[field] = process_column(keyword, det_info) + detectors = [section for section in content if section.endswith("_PRIMARY")] + for detector in detectors: + det_exposure_rec = dict() + det_info = info.copy() + ccdname = f"{detector[0:3]}_{detector[3:6]}" + det_info["ccdname"] = ccdname + det_info["detector"] = camera[ccdname].getId() + for header in content[detector]: + det_info[header["keyword"]] = header["value"] + for field, keyword in det_mapping.items(): + det_exposure_rec[field] = process_column(keyword, det_info) def process_date(day_obs: str) -> None: @@ -261,28 +340,38 @@ def get_kafka_config() -> KafkaConfig: logging.basicConfig(stream=sys.stderr, level=logging.INFO) -instrument = os.environ.get("INSTRUMENT", "LATISS") +instrument = os.environ["INSTRUMENT"] +# Cheat, since we know these are all derived from the same Instrument +formatter = LsstCamRawFormatter match instrument: case "LATISS": from lsst.obs.lsst.translators import LatissTranslator translator = LatissTranslator instrument_mapping = LATISS_MAPPING + det_mapping = DETECTOR_MAPPING + camera = lsst.obs.lsst.Latiss.getCamera() case "LSSTComCam": from lsst.obs.lsst.translators import LsstComCamTranslator translator = LsstComCamTranslator - instrument_mapping = LSSTCOMCAM_MAPPING + instrument_mapping = LSSTCAM_MAPPING + det_mapping = DETECTOR_MAPPING + camera = lsst.obs.lsst.LsstComCam.getCamera() case "LSSTComCamSim": from lsst.obs.lsst.translators import LsstComCamSimTranslator translator = LsstComCamSimTranslator - instrument_mapping = LSSTCOMCAMSIM_MAPPING + instrument_mapping = LSSTCAM_MAPPING + det_mapping = DETECTOR_MAPPING + camera = lsst.obs.lsst.LsstComCamSim.getCamera() case "LSSTCam": from lsst.obs.lsst.translators import LsstCamTranslator translator = LsstCamTranslator instrument_mapping = LSSTCAM_MAPPING + det_mapping = DETECTOR_MAPPING + camera = lsst.obs.lsst.LsstCam.getCamera() logging.info(f"Instrument = {instrument}") engine = setup_postgres() From 5d978e2ed60977fddb63247d39ccae0e0fbe5577 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Sun, 19 May 2024 18:13:45 -0700 Subject: [PATCH 3/5] Add insert_multiple(). Also fix some redundancies and add validation in insert(). --- python/lsst/consdb/pqserver.py | 101 ++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 13 deletions(-) diff --git a/python/lsst/consdb/pqserver.py b/python/lsst/consdb/pqserver.py index 38f94024..0ec752b2 100644 --- a/python/lsst/consdb/pqserver.py +++ b/python/lsst/consdb/pqserver.py @@ -30,7 +30,7 @@ OBS_TYPE_LIST = ["exposure", "visit", "ccdexposure", "ccdvisit"] DTYPE_LIST = ["bool", "int", "float", "str"] -OBS_ID_COLNAME_LIST = ["obs_id", "exposure_id", "ccd_exposure_id", "reb_exposure_id"] +OBS_ID_COLNAME_LIST = ["ccdexposure_id", "exposure_id", "obs_id"] #################### # Global app setup # @@ -582,22 +582,22 @@ def insert_flexible_metadata( } -@app.post("/consdb/insert/") -def insert(instrument: str) -> dict[str, Any] | tuple[dict[str, str], int]: +@app.post("/consdb/insert///obs/") +def insert(instrument: str, table: str, obs_id: int) -> dict[str, Any] | tuple[dict[str, str], int]: """Insert or update column/value pairs in a ConsDB table. Parameters ---------- instrument: `str` Name of the instrument (e.g. ``LATISS``). + table: `str` + Name of table to insert into. + obs_id: `int` + Unique observation identifier. u: `str` Allow update if set to "1" (URL query parameter). - table: `str` - Name of table to insert into (JSON POST data). values: `dict` [ `str`, `Any` ] Dictionary of key/value pairs to insert or update (JSON POST data). - obs_id: `int` - Unique observation identifier (JSON POST data). Returns ------- @@ -616,30 +616,105 @@ def insert(instrument: str) -> dict[str, Any] | tuple[dict[str, str], int]: instrument = instrument.lower() if instrument not in instrument_tables.schemas: raise BadValueException("instrument", instrument, list(instrument_tables.schemas.keys())) - info = _check_json(request.json, "insert", ("table", "values", "obs_id")) - table_name = f"cdb_{instrument}." + info["table"].lower() - table = instrument_tables.schemas[instrument].tables[table_name] + info = _check_json(request.json, "insert", ("values",)) + schema = f"cdb_{instrument}." + table_name = table.lower() + if not table.lower().startswith(schema): + table_name = schema + table_name + table_obj = instrument_tables.schemas[instrument].tables[table_name] valdict = info["values"] obs_id_colname = instrument_tables.obs_id_column[instrument][table_name] - valdict[obs_id_colname] = info["obs_id"] + valdict[obs_id_colname] = obs_id stmt: sqlalchemy.sql.dml.Insert if request.args and request.args.get("u") == "1": stmt = ( - sqlalchemy.dialects.postgresql.insert(table) + sqlalchemy.dialects.postgresql.insert(table_obj) .values(valdict) .on_conflict_do_update(index_elements=[obs_id_colname], set_=valdict) ) else: - stmt = sqlalchemy.insert(table).values(valdict) + stmt = sqlalchemy.insert(table_obj).values(valdict) logger.debug(str(stmt)) with engine.connect() as conn: _ = conn.execute(stmt) conn.commit() + return { + "message": "Data inserted", + "instrument": instrument, + "table": table_name, + "obs_id": obs_id, + } + + +@app.post("/consdb/insert//
") +def insert_multiple(instrument: str, table: str) -> dict[str, Any] | tuple[dict[str, str], int]: + """Insert or update multiple observations in a ConsDB table. + + Parameters + ---------- + instrument: `str` + Name of the instrument (e.g. ``LATISS``). + table: `str` + Name of table to insert into. + u: `str` + Allow update if set to "1" (URL query parameter). + obs_dict: `dict` [ `int`, `dict` [ `str`, `Any` ] ] + Dictionary of unique observation ids and key/value pairs to insert or + update (JSON POST data). + + Returns + ------- + json_dict: `dict` [ `str`, `Any` ] + JSON response with 200 HTTP status on success. + + Raises + ------ + BadJsonException + Raised if JSON is absent or missing a required key. + + BadValueException + Raised if instrument or observation type is invalid. + """ + logger.info(f"{request} {request.json}") + instrument = instrument.lower() + if instrument not in instrument_tables.schemas: + raise BadValueException("instrument", instrument, list(instrument_tables.schemas.keys())) + info = _check_json(request.json, "insert", ("obs_dict")) + schema = f"cdb_{instrument}." + table_name = table.lower() + if not table.lower().startswith(schema): + table_name = schema + table_name + table_obj = instrument_tables.schemas[instrument].tables[table_name] + table_name = f"cdb_{instrument}." + info["table"].lower() + table = instrument_tables.schemas[instrument].tables[table_name] + obs_id_colname = instrument_tables.obs_id_column[instrument][table_name] + + with engine.connect() as conn: + for obs_id, valdict in info["obs_dict"]: + if not isinstance(obs_id, int): + raise BadValueException("obs_id value", obs_id) + valdict[obs_id_colname] = obs_id + + stmt: sqlalchemy.sql.dml.Insert + if request.args and request.args.get("u") == "1": + stmt = ( + sqlalchemy.dialects.postgresql.insert(table_obj) + .values(valdict) + .on_conflict_do_update(index_elements=[obs_id_colname], set_=valdict) + ) + else: + stmt = sqlalchemy.insert(table_obj).values(valdict) + logger.debug(str(stmt)) + # TODO: optimize as executemany + _ = conn.execute(stmt) + conn.commit() + return { "message": "Data inserted", "table": table_name, "instrument": instrument, + "obs_ids": info["obs_dict"].keys(), } From 3b8e0975c30fc8a82610aa1c658b5b2d3e3caf04 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Thu, 23 May 2024 03:21:40 -0700 Subject: [PATCH 4/5] Implement schema overloads. --- python/lsst/consdb/pqserver.py | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/python/lsst/consdb/pqserver.py b/python/lsst/consdb/pqserver.py index 0ec752b2..4ab0c17e 100644 --- a/python/lsst/consdb/pqserver.py +++ b/python/lsst/consdb/pqserver.py @@ -756,6 +756,53 @@ def query() -> dict[str, Any] | tuple[dict[str, str], int]: return result +@app.get("/consdb/schema") +def list_instruments() -> list[str]: + """Retrieve the list of instruments available in ConsDB." + + Returns + ------- + json_list: `list` [ `str` ] + JSON response with 200 HTTP status on success. + Response is a list of instrument names. + + Raises + ------ + BadValueException + Raised if instrument is invalid. + """ + logger.info(request) + return list(instrument_tables.schemas.keys()) + + +@app.get("/consdb/schema/") +def list_table(instrument: str) -> list[str]: + """Retrieve the list of tables for an instrument. + + Parameters + ---------- + instrument: `str` + Name of the instrument (e.g. ``LATISS``). + + Returns + ------- + json_list: `list` [ `str` ] + JSON response with 200 HTTP status on success. + Response is a list of table names. + + Raises + ------ + BadValueException + Raised if instrument is invalid. + """ + logger.info(request) + instrument = instrument.lower() + if instrument not in instrument_tables.schemas: + raise BadValueException("instrument", instrument, list(instrument_tables.schemas.keys())) + schema = instrument_tables.schemas[instrument] + return list(schema.tables.keys()) + + @app.get("/consdb/schema//
") def schema(instrument: str, table: str) -> dict[str, list[str]]: """Retrieve the descriptions of columns in a ConsDB table. From e87687f84b823923bb6c16ead70f5b2d337409e0 Mon Sep 17 00:00:00 2001 From: Kian-Tat Lim Date: Thu, 23 May 2024 04:32:18 -0700 Subject: [PATCH 5/5] Add wide view query. --- python/lsst/consdb/hinfo.py | 61 ++++++++++++++++++------ python/lsst/consdb/pqserver.py | 85 ++++++++++++++++++++++++++++++++-- 2 files changed, 130 insertions(+), 16 deletions(-) diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index 00c4d8ed..b14d1685 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -56,7 +56,7 @@ def region(ra: float, dec: float, rotpa: float, points: list[tuple[str, int, int for detector, offset_x, offset_y in points: skywcs = formatter.makeRawSkyWcsFromBoresight( lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees), - rotpa*lsst.geom.degrees, + rotpa * lsst.geom.degrees, camera[detector], ) bbox = camera[detector].getBBox() @@ -75,9 +75,17 @@ def ccdexposure_id(exposure_id: int, detector: int) -> int: def ccd_region(imgtype: str, ra: float, dec: float, rotpa: float, ccdname: str) -> str | None: if imgtype != "OBJECT": return None - return region(ra, dec, rotpa, [ - (ccdname, 0, 0), (ccdname, 1, 0), (ccdname, 1, 1), (ccdname, 0, 1), - ]) + return region( + ra, + dec, + rotpa, + [ + (ccdname, 0, 0), + (ccdname, 1, 0), + (ccdname, 1, 1), + (ccdname, 0, 1), + ], + ) def fp_region(imgtype: str, ra: float, dec: float, rotpa: float) -> str | None: @@ -86,21 +94,48 @@ def fp_region(imgtype: str, ra: float, dec: float, rotpa: float) -> str | None: return None if instrument == "LATISS": corners = [ - ("RXX_S00", 0, 0), ("RXX_S00", 1, 0), ("RXX_S00", 1, 1), ("RXX_S00", 0, 1), + ("RXX_S00", 0, 0), + ("RXX_S00", 1, 0), + ("RXX_S00", 1, 1), + ("RXX_S00", 0, 1), ] elif instrument == "LSSTComCam" or instrument == "LSSTComCamSim": corners = [ - ("R22_S00", 0, 0), ("R22_S02", 1, 0), ("R22_S20", 0, 1), ("R22_S22", 1, 1), + ("R22_S00", 0, 0), + ("R22_S02", 1, 0), + ("R22_S20", 0, 1), + ("R22_S22", 1, 1), ] elif instrument == "LSSTCam": corners = [ - ("R01_S00", 0, 0), ("R03_S02", 1, 0), ("R03_S02", 1, 1), ("R04_SG1", 1, 0), - ("R04_SG1", 1, 1), ("R04_SG0", 1, 0), ("R04_SG0", 1, 1), ("R14_S02", 1, 0), - ("R34_S22", 1, 1), ("R34_S22", 0, 1), ("R44_SG1", 1, 1), ("R44_SG1", 0, 1), - ("R44_SG0", 1, 1), ("R44_SG0", 0, 1), ("R43_S22", 1, 1), ("R41_S20", 0, 1), - ("R41_S20", 0, 0), ("R40_SG1", 0, 1), ("R40_SG1", 0, 0), ("R40_SG0", 0, 1), - ("R40_SG0", 0, 0), ("R30_S20", 0, 1), ("R10_S00", 0, 0), ("R10_S00", 1, 0), - ("R00_SG1", 0, 0), ("R00_SG1", 1, 0), ("R00_SG0", 0, 0), ("R00_SG0", 1, 0), + ("R01_S00", 0, 0), + ("R03_S02", 1, 0), + ("R03_S02", 1, 1), + ("R04_SG1", 1, 0), + ("R04_SG1", 1, 1), + ("R04_SG0", 1, 0), + ("R04_SG0", 1, 1), + ("R14_S02", 1, 0), + ("R34_S22", 1, 1), + ("R34_S22", 0, 1), + ("R44_SG1", 1, 1), + ("R44_SG1", 0, 1), + ("R44_SG0", 1, 1), + ("R44_SG0", 0, 1), + ("R43_S22", 1, 1), + ("R41_S20", 0, 1), + ("R41_S20", 0, 0), + ("R40_SG1", 0, 1), + ("R40_SG1", 0, 0), + ("R40_SG0", 0, 1), + ("R40_SG0", 0, 0), + ("R30_S20", 0, 1), + ("R10_S00", 0, 0), + ("R10_S00", 1, 0), + ("R00_SG1", 0, 0), + ("R00_SG1", 1, 0), + ("R00_SG0", 0, 0), + ("R00_SG0", 1, 0), ] else: return None diff --git a/python/lsst/consdb/pqserver.py b/python/lsst/consdb/pqserver.py index 4ab0c17e..3b3106d6 100644 --- a/python/lsst/consdb/pqserver.py +++ b/python/lsst/consdb/pqserver.py @@ -27,10 +27,10 @@ from utils import setup_logging, setup_postgres INSTRUMENT_LIST = ["latiss"] -OBS_TYPE_LIST = ["exposure", "visit", "ccdexposure", "ccdvisit"] +OBS_TYPE_LIST = ["exposure", "visit1", "ccdexposure", "ccdvisit1"] DTYPE_LIST = ["bool", "int", "float", "str"] -OBS_ID_COLNAME_LIST = ["ccdexposure_id", "exposure_id", "obs_id"] +OBS_ID_COLNAME_LIST = ["ccdvisit_id", "visit_id", "ccdexposure_id", "exposure_id", "obs_id"] #################### # Global app setup # @@ -73,6 +73,7 @@ def __init__(self): if table_name in md.tables and schema_table_name in md.tables: schema_table = md.tables[schema_table_name] stmt = sqlalchemy.select(schema_table.c["key", "dtype", "doc", "unit", "ucd"]) + logger.debug(str(stmt)) schema = dict() with engine.connect() as conn: for row in conn.execute(stmt): @@ -83,6 +84,7 @@ def refresh_flexible_metadata_schema(self, instrument: str, obs_type: str): schema = dict() schema_table = self.get_flexible_metadata_schema(instrument, obs_type) stmt = sqlalchemy.select(schema_table.c["key", "dtype", "doc", "unit", "ucd"]) + logger.debug(str(stmt)) with engine.connect() as conn: for row in conn.execute(stmt): schema[row[0]] = row[1:] @@ -184,6 +186,38 @@ def get_flexible_metadata_schema(self, instrument: str, obs_type: str): table_name = self.compute_flexible_metadata_table_schema_name(instrument, obs_type) return self.schemas[instrument].tables[table_name] + def compute_wide_view_name(self, instrument: str, obs_type: str) -> str: + """Compute the name of a wide view. + + The wide view joins all tables for a given instrument and observation + type. + + Parameters + ---------- + instrument: `str` + Name of the instrument (e.g. ``LATISS``). + obs_type: `str` + Name of the observation type (e.g. ``Exposure``). + + Returns + ------- + view_nae: `str` + Name of the appropriate wide view. + """ + instrument = instrument.lower() + obs_type = obs_type.lower() + if instrument not in self.schemas: + raise BadValueException("instrument", instrument, list(self.schemas.keys())) + view_name = f"cdb_{instrument}.{obs_type}_wide_view" + if view_name not in self.schemas[instrument].tables: + obs_type_list = [ + name[len(f"cdb_{instrument}.") : -len("_wide_view")] # noqa: E203 + for name in self.schemas[instrument].tables + if name.endswith("_wide_view") + ] + raise BadValueException("observation type", obs_type, obs_type_list) + return view_name + instrument_tables = InstrumentTables() @@ -718,6 +752,51 @@ def insert_multiple(instrument: str, table: str) -> dict[str, Any] | tuple[dict[ } +@app.get("/consdb/query///obs/") +def get_all_metadata( + instrument: str, obs_type: str, obs_id: int +) -> dict[str, Any] | tuple[dict[str, str], int]: + """Get all information about an observation. + + Parameters + ---------- + instrument: `str` + Name of the instrument (e.g. ``LATISS``). + obs_type: `str` + Name of the observation type (e.g. ``Exposure``). + obs_id: `int` + Unique observation identifier. + flex: `str` + Include flexible metadata if set to "1" (URL query parameter). + + Returns + ------- + json_dict: `dict` [ `str`, `Any` ] + JSON response with 200 HTTP status on success. + Response is a dict with columns as keys. + + Raises + ------ + """ + logger.info(request) + instrument = instrument.lower() + obs_type = obs_type.lower() + view_name = instrument_tables.compute_wide_view_name(instrument, obs_type) + view = instrument_tables.schemas[view_name] + obs_id_column = instrument_tables.obs_id_column[instrument][view_name] + stmt = sqlalchemy.select(view).where(view.c[obs_id_column] == obs_id) + logger.debug(str(stmt)) + result = dict() + with engine.connect() as conn: + rows = conn.execute(stmt).all() + assert len(rows) == 1 + result = dict(rows[0]._mapping) + if request.args and "flex" in request.args and request.args["flex"] == "1": + flex_result = get_flexible_metadata(instrument, obs_type, obs_id) + result.update(flex_result) + return result + + @app.post("/consdb/query") def query() -> dict[str, Any] | tuple[dict[str, str], int]: """Query the ConsDB database. @@ -745,7 +824,7 @@ def query() -> dict[str, Any] | tuple[dict[str, str], int]: with engine.connect() as conn: cursor = conn.exec_driver_sql(info["query"]) first = True - result = {} + result: dict[str, Any] = {} rows = [] for row in cursor: if first: