From 09cb45f8cd2e5bfe2c056f06b20464a0bf638115 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Wed, 7 Aug 2024 14:41:51 -0700 Subject: [PATCH 1/8] Set up hinfo for unit testing --- Dockerfile.hinfo | 11 ++++++++--- Dockerfile.pytest | 12 ++++++++++++ python/lsst/consdb/hinfo.py | 20 ++++++++++++++------ 3 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 Dockerfile.pytest diff --git a/Dockerfile.hinfo b/Dockerfile.hinfo index 1a19a73a..d4faa1d4 100644 --- a/Dockerfile.hinfo +++ b/Dockerfile.hinfo @@ -1,9 +1,14 @@ ARG OBS_LSST_VERSION=w_2024_21 FROM lsstsqre/centos:7-stack-lsst_distrib-${OBS_LSST_VERSION} USER lsst + RUN source loadLSST.bash && mamba install aiokafka httpx -RUN source loadLSST.bash && pip install kafkit -COPY python/lsst/consdb/hinfo.py python/lsst/consdb/utils.py ./hinfo/ +RUN source loadLSST.bash && pip install kafkit aiokafka httpx + +WORKDIR /home/lsst/ +COPY --chown=lsst . ./consdb/ +WORKDIR /home/lsst/consdb/ +RUN source /opt/lsst/software/stack/loadLSST.bash && pip install -e . # Environment variables that must be set: # INSTRUMENT: LATISS, LSSTComCam, LSSTCam @@ -16,4 +21,4 @@ COPY python/lsst/consdb/hinfo.py python/lsst/consdb/utils.py ./hinfo/ # KAFKA_GROUP_ID: name of consumer group, default is "consdb-consumer" # KAFKA_USERNAME: username for SASL_PLAIN authentication, default is "consdb" -ENTRYPOINT [ "bash", "-c", "source loadLSST.bash; setup obs_lsst; python ./hinfo/hinfo.py" ] +ENTRYPOINT [ "bash", "-c", "source /opt/lsst/software/stack/loadLSST.bash; setup obs_lsst; python -m lsst.consdb.hinfo" ] diff --git a/Dockerfile.pytest b/Dockerfile.pytest new file mode 100644 index 00000000..edab7cd2 --- /dev/null +++ b/Dockerfile.pytest @@ -0,0 +1,12 @@ +ARG OBS_LSST_VERSION=w_2024_21 +FROM lsstsqre/centos:7-stack-lsst_distrib-${OBS_LSST_VERSION} +USER lsst +RUN source loadLSST.bash && mamba install aiokafka httpx +RUN source loadLSST.bash && pip install kafkit aiokafka httpx pytest-asyncio + +WORKDIR /home/lsst/ +COPY --chown=lsst . ./consdb/ +WORKDIR /home/lsst/consdb/ +RUN source /opt/lsst/software/stack/loadLSST.bash && pip install -e . + +ENTRYPOINT [ "/bin/bash", "-c", "source /opt/lsst/software/stack/loadLSST.bash; setup obs_lsst; pytest ." ] diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index 8776ae96..a9c73dff 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -19,7 +19,8 @@ from lsst.resources import ResourcePath from sqlalchemy import MetaData, Table from sqlalchemy.dialects.postgresql import insert -from utils import setup_logging, setup_postgres + +from .utils import setup_logging, setup_postgres if TYPE_CHECKING: import lsst.afw.cameraGeom # type: ignore @@ -480,10 +481,7 @@ def __init__(self, instrument_name, translator, instrument_mapping, det_mapping, ################# -async def main() -> None: - """Handle Header Service largeFileObjectAvailable messages.""" - global logger, instrument, bucket_prefix, TOPIC_MAPPING - +def get_instrument_dict(instrument: str) -> dict: if instrument == "LATISS": instrument_dict = { "O": Instrument( @@ -531,6 +529,15 @@ async def main() -> None: else: raise ValueError("Unrecognized instrument: {instrument}") + return instrument_dict + + +async def main() -> None: + """Handle Header Service largeFileObjectAvailable messages.""" + global logger, instrument, bucket_prefix, TOPIC_MAPPING + + instrument_dict = get_instrument_dict(instrument) + topic = f"lsst.sal.{TOPIC_MAPPING[instrument]}.logevent_largeFileObjectAvailable" kafka_config = get_kafka_config() async with httpx.AsyncClient() as client: @@ -570,4 +577,5 @@ async def main() -> None: await consumer.stop() -asyncio.run(main()) +if __name__ == "__main__": + asyncio.run(main()) From a1c5a95617bf8909dbf7ee6cf20d62ad231a06c3 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Wed, 7 Aug 2024 17:20:36 -0700 Subject: [PATCH 2/8] Implement a test for hinfo --- python/lsst/consdb/hinfo.py | 10 +- ...erService_header_AT_O_20240801_000302.yaml | 1164 +++++++++++++++++ tests/cdb_latiss.sql | 250 +++- tests/test_hinfo.py | 69 + 4 files changed, 1456 insertions(+), 37 deletions(-) create mode 100644 tests/ATHeaderService_header_AT_O_20240801_000302.yaml create mode 100644 tests/test_hinfo.py diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index a9c73dff..7525ea4e 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -443,15 +443,16 @@ def get_kafka_config() -> KafkaConfig: logger = setup_logging("consdb.hinfo") -instrument = os.environ["INSTRUMENT"] -logger.info(f"Instrument = {instrument}") +instrument = "" +if "INSTRUMENT" in os.environ: + instrument = os.environ["INSTRUMENT"] + logger.info(f"Instrument = {instrument}") bucket_prefix = os.environ.get("BUCKET_PREFIX", "") if bucket_prefix: os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1" -engine = setup_postgres() - +engine = None @dataclass class Instrument: @@ -578,4 +579,5 @@ async def main() -> None: if __name__ == "__main__": + engine = setup_postgres() asyncio.run(main()) diff --git a/tests/ATHeaderService_header_AT_O_20240801_000302.yaml b/tests/ATHeaderService_header_AT_O_20240801_000302.yaml new file mode 100644 index 00000000..f4eea930 --- /dev/null +++ b/tests/ATHeaderService_header_AT_O_20240801_000302.yaml @@ -0,0 +1,1164 @@ +PRIMARY: +- keyword: SIMPLE + value: true + comment: '' +- keyword: BITPIX + value: 8 + comment: bits per data value +- keyword: NAXIS + value: 0 + comment: number of axes +- keyword: EXTEND + value: true + comment: Extensions are permitted +- keyword: ORIGIN + value: LSST DM Header Service + comment: FITS file originator +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Date, night and basic image information ----' +- keyword: TIMESYS + value: TAI + comment: The time scale used +- keyword: DATE + value: '2024-08-01T17:20:20.845' + comment: Creation Date and Time of File +- keyword: DATE-OBS + value: '2024-08-01T17:20:53.970' + comment: Date of the observation (image acquisition) +- keyword: DATE-BEG + value: '2024-08-01T17:20:53.970' + comment: Time at the start of integration +- keyword: DATE-END + value: '2024-08-01T17:20:55.134' + comment: end date of the observation +- keyword: MJD + value: 60523.72246347917 + comment: Modified Julian Date that the file was written +- keyword: MJD-OBS + value: 60523.72284687701 + comment: Modified Julian Date of observation +- keyword: MJD-BEG + value: 60523.72284687701 + comment: Modified Julian Date derived from DATE-BEG +- keyword: MJD-END + value: 60523.722860342255 + comment: Modified Julian Date derived from DATE-END +- keyword: OBSID + value: AT_O_20240801_000302 + comment: ImageName from Camera StartIntergration +- keyword: GROUPID + value: BL271_O_20240801_000001 + comment: '' +- keyword: IMGTYPE + value: FLAT + comment: BIAS, DARK, FLAT, OBJECT +- keyword: BUNIT + value: 'adu ' + comment: Brightness units for pixel array +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Telescope info, location, observer ----' +- keyword: FACILITY + value: Vera C. Rubin Observatory + comment: Facility name +- keyword: TELESCOP + value: LSST AuxTelescope + comment: Telescope name +- keyword: INSTRUME + value: LATISS + comment: Instrument used to obtain these data +- keyword: OBSERVER + value: LSST + comment: Observer name(s) +- keyword: OBS-LONG + value: -70.7476862 + comment: '[deg] Observatory east longitude' +- keyword: OBS-LAT + value: -30.2447889 + comment: '[deg] Observatory latitude' +- keyword: OBS-ELEV + value: 2663.0 + comment: '[m] Observatory elevation' +- keyword: OBSGEO-X + value: 1819093.52 + comment: '[m] X-axis Geocentric coordinate' +- keyword: OBSGEO-Y + value: -5208408.1 + comment: '[m] Y-axis Geocentric coordinate' +- keyword: OBSGEO-Z + value: 3195186.45 + comment: '[m] Z-axis Geocentric coordinate' +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Pointing info, etc. ----' +- keyword: RA + value: 0.0 + comment: RA commanded from pointing component +- keyword: DEC + value: 0.0 + comment: DEC commanded from pointing component +- keyword: RASTART + value: 103.26093940689663 + comment: RA of telescope from AZSTART and ELSTART +- keyword: DECSTART + value: -78.68457809075902 + comment: DEC of telescope from AZSTART and ELSTART +- keyword: RAEND + value: 103.2779105541521 + comment: RA of telescope from AZEND and ELEND +- keyword: DECEND + value: -78.68453411952784 + comment: DEC of telescope from AZEND and ELEND +- keyword: ROTPA + value: 2.4620564838684063 + comment: '[deg] Rotation angle relative to the sky' +- keyword: ROTCOORD + value: sky + comment: Telescope Rotation Coordinates +- keyword: HASTART + value: 0.0 + comment: '[h] Telescope hour angle at start' +- keyword: ELSTART + value: 38.960695248799894 + comment: '[deg] Telescope zenith distance at start' +- keyword: AZSTART + value: -171.23850096695978 + comment: '[deg] Telescope azimuth angle at start' +- keyword: AMSTART + value: 0.0 + comment: Airmass at start +- keyword: HAEND + value: 0.0 + comment: '[h] Telescope hour angle at end' +- keyword: ELEND + value: 38.960736728966296 + comment: '[deg] Telescope zenith distance at end' +- keyword: AZEND + value: -171.23855422350564 + comment: '[deg] Telescope azimuth angle at end' +- keyword: AMEND + value: 0.0 + comment: Airmass at end +- keyword: INSTPORT + value: 2 + comment: The instrument port where LATISS is installed +- keyword: ATM3PORT + value: null + comment: The port the M3 is directing light into +- keyword: TRACKSYS + value: '' + comment: Tracking coordinate system from ATMCS +- keyword: RADESYS + value: '' + comment: Equatorial coordinate system +- keyword: FOCUSZ + value: 0.0 + comment: '[mm] Focus Z position from ATAOS/ATHexapod' +- keyword: OBJECT + value: FlatField position + comment: Name of the observed object +- keyword: DOMEAZ + value: 2.58 + comment: '[deg] Dome Azimuth Position' +- keyword: SHUTLOWR + value: 0.0 + comment: Dome Dropout Door Opening Percentage +- keyword: SHUTUPPR + value: 0.0 + comment: Dome Main Door Opening Percentage +- keyword: VIGNETTE + value: FULLY + comment: Is the telescope vignetted? +- keyword: VIGN_MIN + value: FULLY + comment: Minimum vignetting +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Image-identifying used to build OBS-ID ----' +- keyword: CAMCODE + value: AT + comment: The code for the camera (AT/CC/MC) +- keyword: CONTRLLR + value: O + comment: The controller (e.g. O for OCS, C for (A)CCS) +- keyword: DAYOBS + value: '20240801' + comment: The observation day as defined by image name +- keyword: SEQNUM + value: 302 + comment: The sequence number from the image name +- keyword: TESTTYPE + value: FLAT + comment: 'Test Type: BIAS:DARK:FLAT:OBS:PPUMP:QE:SFLAT' +- keyword: EMUIMAGE + value: '' + comment: Image being emulated +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Additional Keys Information from Camera ----' +- keyword: PROGRAM + value: BLOCK-271 + comment: Name of the program +- keyword: REASON + value: BLOCK-271_1 + comment: Reason for observation +- keyword: OBSANNOT + value: '' + comment: Observing annotation +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Image sequence numbers' +- keyword: CURINDEX + value: 1 + comment: Index number for exposure within the sequence +- keyword: MAXINDEX + value: 1 + comment: Number of requested images in sequence +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Test Stand information ----' +- keyword: TSTAND + value: null + comment: The Test Stand used +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Information from Camera (Common block) ----' +- keyword: IMAGETAG + value: 47798a5397459d9a + comment: DAQ Image id +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Information from Camera ----' +- keyword: TEMP_SET + value: -104.14999999999998 + comment: Temperature set point (deg C) +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Filter/grating information ----' +- keyword: FILTBAND + value: r + comment: Name of the filter band +- keyword: FILTER + value: SDSSr_65mm + comment: Name of the physical filter +- keyword: FILTSLOT + value: 2 + comment: Filter home slot (1-indexed) +- keyword: GRATING + value: empty_1 + comment: Name of physical grating +- keyword: GRATBAND + value: EMPTY + comment: Descriptive name +- keyword: GRATSLOT + value: 0 + comment: Grating's slot (1-indexed) +- keyword: LINSPOS + value: 67.0 + comment: Position of slide +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Exposure-related information ----' +- keyword: EXPTIME + value: 0.93 + comment: Exposure time in seconds +- keyword: SHUTTIME + value: 0.93 + comment: Shutter exposure time in seconds +- keyword: DARKTIME + value: 1.1633970737457275 + comment: Dark time in seconds +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Weather information ----' +- keyword: AIRTEMP + value: 8.949999809265137 + comment: '[degC] Air Temperature' +- keyword: PRESSURE + value: 73580.0 + comment: '[Pascals] Atmospheric Pressure' +- keyword: HUMIDITY + value: 28.0 + comment: Percentage of Relative Humidity +- keyword: WINDSPD + value: 18.59819984436035 + comment: '[m/s] Wind Speed' +- keyword: WINDDIR + value: 13.384214401245117 + comment: '[deg] Wind Direction' +- keyword: SEEING + value: null + comment: '[arcsec] Seeing from FWHM DIMM measurement' +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Header information ----' +- keyword: FILENAME + value: AT_O_20240801_000302.fits + comment: Original file name +- keyword: HEADVER + value: 3.4.5 + comment: Version of header +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Checksums ----' +- keyword: CHECKSUM + value: null + comment: checksum for the current HDU +- keyword: DATASUM + value: null + comment: checksum of the data records +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Hierarch information for CSC Simulatiom Mode ----' +- keyword: HIERARCH SIMULATE ATMCS + value: null + comment: ATMCS Simulation Mode (False=0) +- keyword: HIERARCH SIMULATE ATHEXAPOD + value: 0 + comment: ATHexapod Simulation Mode (False=0) +- keyword: HIERARCH SIMULATE ATPNEUMATICS + value: null + comment: ATPneumatics Simulation Mode (False=0) +- keyword: HIERARCH SIMULATE ATDOME + value: 0 + comment: ATDome Simulation Mode (False=0) +- keyword: HIERARCH SIMULATE ATSPECTROGRAPH + value: 0 + comment: ATSpectrograph Simulation Mode (False=0) +R00S00_PRIMARY: +- keyword: null + value: null + comment: '---- Information from Camera per sensor ----' +- keyword: CCD_MANU + value: ITL + comment: CCD Manufacturer +- keyword: CCD_TYPE + value: 2 + comment: CCD Model Number +- keyword: CCD_SERN + value: '20862' + comment: Manufacturers CCD Serial Number +- keyword: CCDTEMP + value: .nan + comment: Measured temperature (deg C) +- keyword: LSST_NUM + value: ITL-3800C-068 + comment: LSST Assigned CCD Number +- keyword: SEQCKSUM + value: '1477162166' + comment: Checksum of Sequencer +- keyword: SEQNAME + value: FP_ITL_2s_ir2_v26_no_RG.seq + comment: SequenceName from Camera StartIntergration +- keyword: REBNAME + value: Unknown + comment: Name of the REB +- keyword: CONTNUM + value: '412227310' + comment: CCD Controller (WREB) Serial Number +- keyword: CCDSLOT + value: S00 + comment: Name of the CCD Slot (SXx) +- keyword: RAFTBAY + value: R00 + comment: Name of the RAFT Bay (Rnn) +- keyword: COMMENT + value: '' + comment: '' +- keyword: null + value: null + comment: '---- Geometry from Camera ----' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: Size of sensor +- keyword: OVERH + value: 64 + comment: Over-scan pixels +- keyword: OVERV + value: 48 + comment: Vert-overscan pix +- keyword: PREH + value: 0 + comment: Pre-scan pixels +R00S00_Segment10: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment10 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[509:1,1:2000]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 510 + comment: detector transformation vector +- keyword: DTV2 + value: 0 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: 1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment11: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment11 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[1018:510,1:2000]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 1019 + comment: detector transformation vector +- keyword: DTV2 + value: 0 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: 1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment12: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment12 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[1527:1019,1:2000]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 1528 + comment: detector transformation vector +- keyword: DTV2 + value: 0 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: 1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment13: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment13 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[2036:1528,1:2000]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 2037 + comment: detector transformation vector +- keyword: DTV2 + value: 0 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: 1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment14: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment14 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[2545:2037,1:2000]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 2546 + comment: detector transformation vector +- keyword: DTV2 + value: 0 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: 1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment15: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment15 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[3054:2546,1:2000]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 3055 + comment: detector transformation vector +- keyword: DTV2 + value: 0 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: 1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment16: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment16 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[3563:3055,1:2000]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 3564 + comment: detector transformation vector +- keyword: DTV2 + value: 0 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: 1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment17: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment17 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[4072:3564,1:2000]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 4073 + comment: detector transformation vector +- keyword: DTV2 + value: 0 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: 1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment07: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment07 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[4072:3564,4000:2001]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 4073 + comment: detector transformation vector +- keyword: DTV2 + value: 4001 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment06: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment06 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[3563:3055,4000:2001]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 3564 + comment: detector transformation vector +- keyword: DTV2 + value: 4001 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment05: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment05 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[3054:2546,4000:2001]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 3055 + comment: detector transformation vector +- keyword: DTV2 + value: 4001 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment04: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment04 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[2545:2037,4000:2001]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 2546 + comment: detector transformation vector +- keyword: DTV2 + value: 4001 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment03: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment03 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[2036:1528,4000:2001]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 2037 + comment: detector transformation vector +- keyword: DTV2 + value: 4001 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment02: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment02 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[1527:1019,4000:2001]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 1528 + comment: detector transformation vector +- keyword: DTV2 + value: 4001 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment01: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment01 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[1018:510,4000:2001]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 1019 + comment: detector transformation vector +- keyword: DTV2 + value: 4001 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix +R00S00_Segment00: +- keyword: XTENSION + value: 'IMAGE ' + comment: marks beginning of new HDU +- keyword: BITPIX + value: 32 + comment: bits per data value +- keyword: INHERIT + value: true + comment: Inherits global header +- keyword: PCOUNT + value: 0 + comment: Required value +- keyword: GCOUNT + value: 1 + comment: Required value +- keyword: EXTNAME + value: Segment00 + comment: '' +- keyword: DATASEC + value: '[1:509,1:2000]' + comment: '' +- keyword: DETSEC + value: '[509:1,4000:2001]' + comment: '' +- keyword: DETSIZE + value: '[1:4072,1:4000]' + comment: '' +- keyword: DTV1 + value: 510 + comment: detector transformation vector +- keyword: DTV2 + value: 4001 + comment: detector transformation vector +- keyword: DTM1_1 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM2_2 + value: -1.0 + comment: detector transformation matrix +- keyword: DTM1_2 + value: 0 + comment: detector transformation matrix +- keyword: DTM2_1 + value: 0 + comment: detector transformation matrix diff --git a/tests/cdb_latiss.sql b/tests/cdb_latiss.sql index aaf25cd8..02557a32 100644 --- a/tests/cdb_latiss.sql +++ b/tests/cdb_latiss.sql @@ -3,44 +3,40 @@ CREATE TABLE main.exposure ( exposure_id BIGINT NOT NULL, exposure_name VARCHAR(20) NOT NULL, controller VARCHAR(1) NOT NULL, - day_obs BIGINT NOT NULL, - seq_num BIGINT NOT NULL, - physical_filter TEXT, - band TEXT, + day_obs INTEGER NOT NULL, + seq_num INTEGER NOT NULL, + physical_filter VARCHAR(32), + band VARCHAR(32), s_ra DOUBLE, s_dec DOUBLE, sky_rotation DOUBLE, - azimuth_start DOUBLE, - azimuth_end DOUBLE, - azimuth DOUBLE, - altitude_start DOUBLE, - altitude_end DOUBLE, - altitude DOUBLE, - zenith_distance_start DOUBLE, - zenith_distance_end DOUBLE, - zenith_distance DOUBLE, - airmass DOUBLE, + azimuth_start FLOAT, + azimuth_end FLOAT, + azimuth FLOAT, + altitude_start FLOAT, + altitude_end FLOAT, + altitude FLOAT, + zenith_distance_start FLOAT, + zenith_distance_end FLOAT, + zenith_distance FLOAT, + airmass FLOAT, exp_midpt TIMESTAMP, exp_midpt_mjd DOUBLE, obs_start TIMESTAMP, obs_start_mjd DOUBLE, obs_end TIMESTAMP, obs_end_mjd DOUBLE, - exp_time DOUBLE, - shut_time DOUBLE, - dark_time DOUBLE, - group_id TEXT, + exp_time FLOAT, + shut_time FLOAT, + dark_time FLOAT, + group_id VARCHAR(64), cur_index INTEGER, max_index INTEGER, - img_type TEXT, + img_type VARCHAR(64), emulated BOOLEAN, - science_program TEXT, - observation_reason TEXT, - target_name TEXT, - shutter_open_begin TIMESTAMP, - shutter_open_end TIMESTAMP, - shutter_close_begin TIMESTAMP, - shutter_close_end TIMESTAMP, + science_program VARCHAR(64), + observation_reason VARCHAR(68), + target_name VARCHAR(64), air_temp FLOAT, pressure FLOAT, humidity FLOAT, @@ -52,26 +48,106 @@ CREATE TABLE main.exposure ( focus_z FLOAT, dome_azimuth FLOAT, simulated BOOLEAN, - PRIMARY KEY (day_obs, seq_num), - CONSTRAINT un_exposure_id UNIQUE (exposure_id) + vignette VARCHAR(10), + vignette_min VARCHAR(10), + s_region VARCHAR(1024), + PRIMARY KEY (exposure_id), + CONSTRAINT un_day_obs_seq_num UNIQUE (day_obs, seq_num) ) ; CREATE TABLE main.exposure_flexdata_schema ( - "key" TEXT NOT NULL, - dtype TEXT NOT NULL, + "key" VARCHAR(128) NOT NULL, + dtype VARCHAR(64) NOT NULL, doc TEXT NOT NULL, - unit TEXT, - ucd TEXT, + unit VARCHAR(128), + ucd VARCHAR(128), PRIMARY KEY ("key") ) ; +CREATE TABLE main.ccdexposure_flexdata_schema ( + "key" VARCHAR(128) NOT NULL, + dtype VARCHAR(64) NOT NULL, + doc TEXT NOT NULL, + unit VARCHAR(128), + ucd VARCHAR(128), + PRIMARY KEY ("key") +) + +; + +CREATE TABLE main.visit1 ( + visit_id BIGINT NOT NULL, + exposure_name VARCHAR(20) NOT NULL, + controller VARCHAR(1) NOT NULL, + day_obs INTEGER NOT NULL, + seq_num INTEGER NOT NULL, + physical_filter VARCHAR(32), + band VARCHAR(32), + s_ra DOUBLE, + s_dec DOUBLE, + sky_rotation DOUBLE, + azimuth_start FLOAT, + azimuth_end FLOAT, + azimuth FLOAT, + altitude_start FLOAT, + altitude_end FLOAT, + altitude FLOAT, + zenith_distance_start FLOAT, + zenith_distance_end FLOAT, + zenith_distance FLOAT, + airmass FLOAT, + exp_midpt TIMESTAMP, + exp_midpt_mjd DOUBLE, + obs_start TIMESTAMP, + obs_start_mjd DOUBLE, + obs_end TIMESTAMP, + obs_end_mjd DOUBLE, + exp_time FLOAT, + shut_time FLOAT, + dark_time FLOAT, + group_id VARCHAR(64), + cur_index INTEGER, + max_index INTEGER, + img_type VARCHAR(64), + emulated BOOLEAN, + science_program VARCHAR(64), + observation_reason VARCHAR(68), + target_name VARCHAR(64), + air_temp FLOAT, + pressure FLOAT, + humidity FLOAT, + wind_speed FLOAT, + wind_dir FLOAT, + dimm_seeing FLOAT, + shut_lower FLOAT, + shut_upper FLOAT, + focus_z FLOAT, + simulated BOOLEAN, + vignette VARCHAR(10), + vignette_min VARCHAR(10), + s_region VARCHAR(1024), + PRIMARY KEY (visit_id) +) + +; + +CREATE TABLE main.ccdvisit1 ( + ccdvisit_id BIGINT NOT NULL, + visit_id BIGINT NOT NULL, + detector INTEGER NOT NULL, + s_region VARCHAR(1024), + PRIMARY KEY (ccdvisit_id) +) + +; + CREATE TABLE main.exposure_flexdata ( obs_id BIGINT NOT NULL, - "key" TEXT NOT NULL, + "key" VARCHAR(128) NOT NULL, value TEXT, PRIMARY KEY (obs_id, "key"), CONSTRAINT fk_obs_id FOREIGN KEY(obs_id) REFERENCES exposure (exposure_id), @@ -79,3 +155,111 @@ CREATE TABLE main.exposure_flexdata ( ) ; + +CREATE TABLE main.ccdexposure ( + ccdexposure_id BIGINT NOT NULL, + exposure_id BIGINT NOT NULL, + detector INTEGER NOT NULL, + s_region VARCHAR(1024), + PRIMARY KEY (ccdexposure_id), + CONSTRAINT un_exposure_id_detector UNIQUE (exposure_id, detector), + CONSTRAINT fk_exposure_id FOREIGN KEY(exposure_id) REFERENCES exposure (exposure_id) +) + +; + +CREATE TABLE main.visit1_quicklook ( + visit_id BIGINT NOT NULL, + n_inputs INTEGER, + astrom_offset_mean FLOAT, + astrom_offset_std FLOAT, + eff_time FLOAT, + eff_time_psf_sigma_scale FLOAT, + eff_time_sky_bg_scale FLOAT, + eff_time_zero_point_scale FLOAT, + max_dist_to_nearest_psf FLOAT, + mean_var FLOAT, + n_psf_star INTEGER, + psf_area FLOAT, + psf_ixx FLOAT, + psf_ixy FLOAT, + psf_iyy FLOAT, + psf_sigma FLOAT, + psf_star_delta_e1_median FLOAT, + psf_star_delta_e1_scatter FLOAT, + psf_star_delta_e2_median FLOAT, + psf_star_delta_e2_scatter FLOAT, + psf_star_delta_size_median FLOAT, + psf_star_delta_size_scatter FLOAT, + psf_star_scaled_delta_size_scatter FLOAT, + psf_trace_radius_delta FLOAT, + sky_bg FLOAT, + sky_noise FLOAT, + seeing_zenith_500nm DOUBLE, + zero_point FLOAT, + low_snr_source_count INTEGER, + high_snr_source_count INTEGER, + PRIMARY KEY (visit_id), + CONSTRAINT fk_obs_id FOREIGN KEY(visit_id) REFERENCES exposure (exposure_id) +) + +; + +CREATE TABLE main.ccdexposure_camera ( + ccdexposure_id BIGINT NOT NULL, + temp_set FLOAT, + ccd_temp FLOAT, + PRIMARY KEY (ccdexposure_id), + CONSTRAINT fk_ccdexposure_id FOREIGN KEY(ccdexposure_id) REFERENCES ccdexposure (ccdexposure_id) +) + +; + +CREATE TABLE main.ccdexposure_flexdata ( + obs_id BIGINT NOT NULL, + "key" VARCHAR(128) NOT NULL, + value TEXT, + PRIMARY KEY (obs_id, "key"), + CONSTRAINT fk_obs_id FOREIGN KEY(obs_id) REFERENCES ccdexposure (ccdexposure_id), + CONSTRAINT fk_key FOREIGN KEY("key") REFERENCES ccdexposure_flexdata_schema ("key") +) + +; + +CREATE TABLE main.ccdvisit1_quicklook ( + ccdvisit_id BIGINT NOT NULL, + s_ra DOUBLE, + s_dec DOUBLE, + zenith_distance FLOAT, + photometric_calib FLOAT, + psf_sigma FLOAT, + sky_bg FLOAT, + sky_noise FLOAT, + zero_point FLOAT, + seeing_zenith_500nm DOUBLE, + astrom_offset_mean FLOAT, + astrom_offset_std FLOAT, + eff_time FLOAT, + eff_time_psf_sigma_scale FLOAT, + eff_time_sky_bg_scale FLOAT, + eff_time_zero_point_scale FLOAT, + mean_var FLOAT, + n_psf_star INTEGER, + psf_area FLOAT, + psf_ixx FLOAT, + psf_ixy FLOAT, + psf_iyy FLOAT, + psf_star_delta_e1_median FLOAT, + psf_star_delta_e2_median FLOAT, + psf_star_delta_e1_scatter FLOAT, + psf_star_delta_e2_scatter FLOAT, + psf_star_delta_size_median FLOAT, + psf_star_delta_size_scatter FLOAT, + psf_star_scaled_delta_size_scatter FLOAT, + psf_trace_radius_delta FLOAT, + max_dist_to_nearest_psf FLOAT, + PRIMARY KEY (ccdvisit_id), + CONSTRAINT fk_obs_id FOREIGN KEY(ccdvisit_id) REFERENCES ccdexposure (ccdexposure_id) +) + +; diff --git a/tests/test_hinfo.py b/tests/test_hinfo.py new file mode 100644 index 00000000..66dbbc60 --- /dev/null +++ b/tests/test_hinfo.py @@ -0,0 +1,69 @@ +import os +import pytest +import sqlite3 +import tempfile +from pathlib import Path +from sqlalchemy import MetaData, Table, select +import yaml + +from lsst.resources import ResourcePath + +from lsst.consdb import hinfo, utils + +@pytest.fixture +def tmpdir(scope="module"): + os.environ["INSTRUMENT"] = "LATISS" + tmpdir = Path(tempfile.mkdtemp()) + return tmpdir + shutil.rmtree(tmpdir) + + +@pytest.fixture +def engine(tmpdir, scope="module"): + db_path = tmpdir / "test.db" + instruments = ["latiss"] + with sqlite3.connect(db_path) as conn: + conn.execute("CREATE TABLE schemas (name text, path text)") + for instrument in instruments: + schema = f"cdb_{instrument}" + sql = Path(__file__).parent / f"{schema}.sql" + if not sql.exists(): + continue + schema_path = tmpdir / f"{schema}.db" + conn.execute(f"INSERT INTO schemas VALUES ('{schema}', '{schema_path}')") + with sqlite3.connect(schema_path) as schema_conn: + schema_conn.executescript(sql.read_text()) + schema_conn.close() + + os.environ["POSTGRES_URL"] = f"sqlite:///{db_path}" + hinfo.engine = utils.setup_postgres() + hinfo.instrument = "LATISS" + + return hinfo.engine + +def _header_lookup(header, key): + for line in header: + if line["keyword"] == key: + return line["value"] + return None + +def test_process_resource(engine): + yaml_path = Path(__file__).parent / "ATHeaderService_header_AT_O_20240801_000302.yaml" + rp = ResourcePath(yaml_path) + + with open(yaml_path, "r") as f: + header = yaml.safe_load(f)["PRIMARY"] + + instrument_dict = hinfo.get_instrument_dict("LATISS") + hinfo.process_resource(rp, instrument_dict) + + metadata_obj = MetaData(schema="cdb_latiss") + exposure_table = Table("exposure", metadata_obj, autoload_with=engine) + with engine.begin() as conn: + row = conn.execute(select(exposure_table)).first() + print(f"{row=}") + print(f"{row.exposure_name=}") + + assert _header_lookup(header, "OBSID") == row.exposure_name + assert _header_lookup(header, "VIGN_MIN") == row.vignette_min + assert _header_lookup(header, "DOMEAZ") == row.dome_azimuth \ No newline at end of file From d809865f0a0f678d32346477cbca7f9fa2b85bab Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Wed, 7 Aug 2024 17:25:04 -0700 Subject: [PATCH 3/8] Add an assert for one of the computed functions --- tests/test_hinfo.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_hinfo.py b/tests/test_hinfo.py index 66dbbc60..b3242171 100644 --- a/tests/test_hinfo.py +++ b/tests/test_hinfo.py @@ -66,4 +66,8 @@ def test_process_resource(engine): assert _header_lookup(header, "OBSID") == row.exposure_name assert _header_lookup(header, "VIGN_MIN") == row.vignette_min - assert _header_lookup(header, "DOMEAZ") == row.dome_azimuth \ No newline at end of file + assert _header_lookup(header, "DOMEAZ") == row.dome_azimuth + + elstart = _header_lookup(header, "ELSTART") + elend = _header_lookup(header, "ELEND") + assert 0.5 * (elstart + elend) == row.zenith_distance \ No newline at end of file From a38ae4ba370e8d545fc9ff44f2eaa1b755fdce1d Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Wed, 7 Aug 2024 17:47:26 -0700 Subject: [PATCH 4/8] Modify git workflow to use Docker container --- .github/workflows/pytest.yaml | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 32382a70..741c2fe7 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -14,25 +14,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: "pip" + - name: Build docker image + run: docker build -f Dockerfile.pytest -t pytest_image . - - name: Editable mode install - run: | - python -m pip install uv - uv pip install --system pytest pytest-cov pytest-html pytest-asyncio httpx - uv pip install --system -e . - - name: Test with pytest - run: | - pytest --cov=./ --cov-report=html --html=pytest_report.html --self-contained-html - - - name: Upload coverage to codecov - uses: codecov/codecov-action@v4 - with: - files: ./coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} + - name: Run docker image + run: docker run --rm -e PYTEST_ADDOPTS="--color=yes" pytest_image From 909e797033dca216a5b8cedcc8b3bb35101e261b Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Wed, 7 Aug 2024 17:56:50 -0700 Subject: [PATCH 5/8] Lint and isort --- python/lsst/consdb/hinfo.py | 1 + tests/test_hinfo.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index 7525ea4e..b05b105b 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -454,6 +454,7 @@ def get_kafka_config() -> KafkaConfig: engine = None + @dataclass class Instrument: instrument_name: str diff --git a/tests/test_hinfo.py b/tests/test_hinfo.py index b3242171..c68edad6 100644 --- a/tests/test_hinfo.py +++ b/tests/test_hinfo.py @@ -1,21 +1,20 @@ import os -import pytest import sqlite3 import tempfile from pathlib import Path -from sqlalchemy import MetaData, Table, select -import yaml +import pytest +import yaml +from lsst.consdb import hinfo, utils from lsst.resources import ResourcePath +from sqlalchemy import MetaData, Table, select -from lsst.consdb import hinfo, utils @pytest.fixture def tmpdir(scope="module"): os.environ["INSTRUMENT"] = "LATISS" tmpdir = Path(tempfile.mkdtemp()) return tmpdir - shutil.rmtree(tmpdir) @pytest.fixture @@ -41,12 +40,14 @@ def engine(tmpdir, scope="module"): return hinfo.engine + def _header_lookup(header, key): for line in header: if line["keyword"] == key: return line["value"] return None + def test_process_resource(engine): yaml_path = Path(__file__).parent / "ATHeaderService_header_AT_O_20240801_000302.yaml" rp = ResourcePath(yaml_path) @@ -70,4 +71,4 @@ def test_process_resource(engine): elstart = _header_lookup(header, "ELSTART") elend = _header_lookup(header, "ELEND") - assert 0.5 * (elstart + elend) == row.zenith_distance \ No newline at end of file + assert 0.5 * (elstart + elend) == row.zenith_distance From bae092ba81a622fff0ace22acdd6b309940a046d Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Fri, 9 Aug 2024 10:03:01 -0700 Subject: [PATCH 6/8] Implement cleanup of tmpdir per suggestion by @vebop --- tests/test_hinfo.py | 11 +++++++++-- tests/test_pqserver.py | 14 ++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/test_hinfo.py b/tests/test_hinfo.py index c68edad6..df28da23 100644 --- a/tests/test_hinfo.py +++ b/tests/test_hinfo.py @@ -2,6 +2,7 @@ import sqlite3 import tempfile from pathlib import Path +import shutil import pytest import yaml @@ -14,7 +15,10 @@ def tmpdir(scope="module"): os.environ["INSTRUMENT"] = "LATISS" tmpdir = Path(tempfile.mkdtemp()) - return tmpdir + try: + yield tmpdir + finally: + shutil.rmtree(tmpdir) @pytest.fixture @@ -38,7 +42,10 @@ def engine(tmpdir, scope="module"): hinfo.engine = utils.setup_postgres() hinfo.instrument = "LATISS" - return hinfo.engine + try: + yield hinfo.engine + finally: + hinfo.engine.dispose() def _header_lookup(header, key): diff --git a/tests/test_pqserver.py b/tests/test_pqserver.py index 95289b5a..c7bde590 100644 --- a/tests/test_pqserver.py +++ b/tests/test_pqserver.py @@ -16,8 +16,10 @@ def _assert_http_status(response: Response, status: int): @pytest.fixture def tmpdir(scope="module"): tmpdir = Path(tempfile.mkdtemp()) - return tmpdir - shutil.rmtree(tmpdir) + try: + yield tmpdir + finally: + shutil.rmtree(tmpdir) @pytest.fixture @@ -53,9 +55,13 @@ def db(tmpdir, scope="module"): @pytest.fixture def app(db, scope="module"): os.environ["POSTGRES_URL"] = f"sqlite:///{db}" - from lsst.consdb import pqserver + from lsst.consdb import pqserver, utils - return pqserver.app + pqserver.engine = utils.setup_postgres() + try: + yield pqserver.app + finally: + pqserver.engine.dispose() @pytest.fixture From 192f8962acc78a03d034b72e2aca6959fd67d70b Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Fri, 9 Aug 2024 10:06:07 -0700 Subject: [PATCH 7/8] isort --- tests/test_hinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_hinfo.py b/tests/test_hinfo.py index df28da23..4dfb4c62 100644 --- a/tests/test_hinfo.py +++ b/tests/test_hinfo.py @@ -1,8 +1,8 @@ import os +import shutil import sqlite3 import tempfile from pathlib import Path -import shutil import pytest import yaml From b3e5cbb0ce9d7500eead2085ba107626b3570d41 Mon Sep 17 00:00:00 2001 From: Brian Brondel Date: Tue, 13 Aug 2024 09:01:27 -0700 Subject: [PATCH 8/8] More idiomatic use of getenv suggested by @ktlim --- python/lsst/consdb/hinfo.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py index b05b105b..271facd7 100644 --- a/python/lsst/consdb/hinfo.py +++ b/python/lsst/consdb/hinfo.py @@ -443,10 +443,8 @@ def get_kafka_config() -> KafkaConfig: logger = setup_logging("consdb.hinfo") -instrument = "" -if "INSTRUMENT" in os.environ: - instrument = os.environ["INSTRUMENT"] - logger.info(f"Instrument = {instrument}") +instrument = os.environ.get("INSTRUMENT", "") +logger.info(f"{instrument=}") bucket_prefix = os.environ.get("BUCKET_PREFIX", "") if bucket_prefix: os.environ["LSST_DISABLE_BUCKET_VALIDATION"] = "1"