Added pre-commit config and run over all files

lsst-dm · May 6, 2024 · 23f43c5 · 23f43c5
1 parent 0ed67df
commit 23f43c5
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 52 deletions.
diff --git a/.tmp/config.yaml b/.tmp/config.yaml
@@ -17,44 +17,44 @@ topics:
 # Were any pointing system warnings triggered during the exposure?
 - name: lsst.sal.ATPtg.logevent_airmassWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_azWrapWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_blindSpotWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_elLimitWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_focusNameInconsistentWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_moonProximityWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_mountDataWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_objectSetWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_rotWrapWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_sunProximityWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.ATPtg.logevent_objectSetWarning
   fields: ["level"]
-  function: mean  
+  function: mean
 - name: lsst.sal.MTAOS.logevent_cameraHexapodCorrection
   fields: ["x", "y", "z", "u", "v", "w"]
-  function: mean  
+  function: mean
 # annularZernikeCoeff: array with z4-z22 terms of annular Zernike polynomials
 - name: lsst.sal.MTAOS.logevent_wavefrontError
   fields: ["sensorId", "annularZernikeCoeff"]
-  function: mean  
+  function: mean
 - name: lsst.sal.DIMM.logevent_dimmMeasurement
   fields: ["fwhm"]
   function: mean
@@ -76,7 +76,7 @@ topics:
   fields: ["gDV", "oDV", "oGV", "rDV", "temp"]
   function: mean
 - name: lsst.sal.CCCamera.focal_plane_Reb
-  fields: ["anaI", "anaV", "aspicl_Temp0", "aspicu_Temp0", "clkHI", "clkHV", "clkLI", "clkLV", "digI", "digV"] 
+  fields: ["anaI", "anaV", "aspicl_Temp0", "aspicu_Temp0", "clkHI", "clkHV", "clkLI", "clkLV", "digI", "digV"]
   function: mean
 exposure_table: exposure
 visit_table: visit
@@ -85,5 +85,5 @@ visit_table: visit
 #                         lsst.sal.WeatherStation.relativeHumidity
 #                         lsst.sal.WeatherStation.windDirection
 # summit-latiss unidentifyed topics by @id:
-#                         "@id": "#ExposureEFD.Camera_alertRaised"                        
+#                         "@id": "#ExposureEFD.Camera_alertRaised"
 #                         "@id": "#ExposureCcdEFD.Camera_oDI" (similar oGV exists in lsst.sal.CCCamera.focal_plane_Ccd)
diff --git a/Dockerfile.server b/Dockerfile.server
@@ -5,4 +5,3 @@ COPY src/server.py /consdb-server/
 # Environment variables that must be set:
 # POSTGRES_URL
 ENTRYPOINT [ "gunicorn", "-b", "0.0.0.0:8000", "-w", "2", "server:app" ]
-
diff --git a/python/lsst/consdb/__init__.py b/python/lsst/consdb/__init__.py
@@ -19,4 +19,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-# from .version import *  # Generated by sconsUtils
+# from .version import *  # Generated by sconsUtils
diff --git a/python/lsst/consdb/client.py b/python/lsst/consdb/client.py
@@ -1,10 +1,11 @@
 import os
-from pandas import DataFrame
-import requests
-from requests.exceptions import RequestException
 from typing import Any, Iterable
 from urllib.parse import urljoin
 
+import requests
+from pandas import DataFrame
+from requests.exceptions import RequestException
+
 session = requests.Session()
 base_url = os.environ["CONSDB_URL"]
 

diff --git a/python/lsst/consdb/hinfo.py b/python/lsst/consdb/hinfo.py
@@ -161,7 +161,7 @@ def process_column(column_def: str | Sequence, info: dict) -> Any:
         The value to use for the column.
         None if any input value is missing.
     """
-    if type(column_def) is str:
+    if isinstance(column_def, str):
         if column_def in info:
             return info[column_def]
     elif type(column_def) is tuple:

diff --git a/python/lsst/consdb/server.py b/python/lsst/consdb/server.py
@@ -1,7 +1,6 @@
-from flask import Flask, request
-from sqlalchemy import create_engine, MetaData
 import sqlalchemy.exc
-
+from flask import Flask, request
+from sqlalchemy import MetaData, create_engine
 
 app = Flask(__name__)
 engine = create_engine("postgresql://usdf-butler.slac.stanford.edu:5432/lsstdb1")
@@ -53,4 +52,6 @@ def query():
 
 @app.get("/schema/<table>")
 def schema(table: str):
-    return [(c.name, str(c.type), c.doc) for c in metadata_obj.tables[table.lower()].columns]
+    return [
+        (c.name, str(c.type), c.doc) for c in metadata_obj.tables[table.lower()].columns
+    ]
diff --git a/python/lsst/consdb/transform_efd.py b/python/lsst/consdb/transform_efd.py
@@ -1,13 +1,14 @@
 import argparse
-from typing import TYPE_CHECKING, Any, Callable, Optional
+import asyncio
+from typing import Any, Callable, Optional
 
 import astropy.time
 import lsst_efd_client
 import pandas
 import yaml
 from lsst.daf.butler import Butler, DimensionRecord
-from sqlalchemy import create_engine, Engine
-import asyncio
+from sqlalchemy import Engine, create_engine
+
 
 class Summary:
     # TODO define summary
@@ -50,9 +51,7 @@ class Records:
     def __init__(self, db: Engine):
         self._db = db
 
-    def add(
-        self, dim: DimensionRecord, topic: dict[str, Any], summary: Any
-    ) -> None:
+    def add(self, dim: DimensionRecord, topic: dict[str, Any], summary: Any) -> None:
         pass
 
     def write(self, table: str) -> None:
@@ -78,7 +77,7 @@ async def get_efd_values(
         end.utc + window,
         topic.get("index", None),
     )
-    series.to_csv('.tmp/' + topic['name'] + '.csv')
+    series.to_csv(".tmp/" + topic["name"] + ".csv")
     print(f"Topic: {topic['name']}")
     # print(f"Window: {window}")
     # print(f"Series: {len(series)}")
@@ -87,27 +86,42 @@ async def get_efd_values(
     return EfdValues(topic, window, series)
 
 
-def get_exposures_by_period(butler: Butler, instrument: str, start: astropy.time.Time, end: astropy.time.Time, limit: Optional[int] = None):
+def get_exposures_by_period(
+    butler: Butler,
+    instrument: str,
+    start: astropy.time.Time,
+    end: astropy.time.Time,
+    limit: Optional[int] = None,
+):
 
-    where_clause = f"instrument=instr and exposure.timespan OVERLAPS (T'{start}', T'{end}')"
+    where_clause = (
+        f"instrument=instr and exposure.timespan OVERLAPS (T'{start}', T'{end}')"
+    )
     return butler.registry.queryDimensionRecords(
-        "exposure", 
-        where=where_clause,
-        bind=dict(instr=instrument)
+        "exposure", where=where_clause, bind=dict(instr=instrument)
     ).limit(limit)
 
-def get_visits_by_period(butler: Butler, instrument: str, start: astropy.time.Time, end: astropy.time.Time, limit: Optional[int] = None):
 
-    where_clause = f"instrument=instr and visit.timespan OVERLAPS (T'{start}', T'{end}')"
+def get_visits_by_period(
+    butler: Butler,
+    instrument: str,
+    start: astropy.time.Time,
+    end: astropy.time.Time,
+    limit: Optional[int] = None,
+):
+
+    where_clause = (
+        f"instrument=instr and visit.timespan OVERLAPS (T'{start}', T'{end}')"
+    )
     return butler.registry.queryDimensionRecords(
-        "visit", 
-        where=where_clause,
-        bind=dict(instr=instrument)
+        "visit", where=where_clause, bind=dict(instr=instrument)
     ).limit(limit)
 
+
 def butler_query_results_to_pandas(query):
     return pandas.DataFrame([q.toDict() for q in query])
 
+
 async def process_interval(
     butler: Butler,
     db: Engine,
@@ -118,7 +132,7 @@ async def process_interval(
     end_time: str,
 ) -> None:
 
-    print(f"Process Interval")   
+    print("Process Interval")
 
     start = astropy.time.Time(start_time, format="isot")
     end = astropy.time.Time(end_time, format="isot")
@@ -135,20 +149,19 @@ async def process_interval(
         if record.timespan.end < end:
             exposure_list.append(record)
             min_topic_time = min(record.timespan.begin, min_topic_time)
-            max_topic_time = max(record.timespan.begin, max_topic_time)        
+            max_topic_time = max(record.timespan.begin, max_topic_time)
 
     for record in get_visits_by_period(butler, instrument, start, end, limit=10):
         if record.timespan.end < end:
             visit_list.append(record)
             min_topic_time = min(record.timespan.begin, min_topic_time)
-            max_topic_time = max(record.timespan.begin, max_topic_time)        
+            max_topic_time = max(record.timespan.begin, max_topic_time)
 
     print(f"Exposures: {len(exposure_list)}")
     print(f"Visits: {len(visit_list)}")
     print(f"Min Topic time: {min_topic_time}")
     print(f"Max Topic time: {max_topic_time}")
 
-
     # exposure_records = Records(db)
     # print("Exposure Records:")
     # print(exposure_records)
@@ -167,7 +180,7 @@ async def process_interval(
 
     for topic in config["topics"]:
         efd_values = await get_efd_values(efd, topic, min_topic_time, max_topic_time)
-        # print(efd_values)
+        print(efd_values)
 
 
 def build_argparser() -> argparse.ArgumentParser:
@@ -192,7 +205,14 @@ def build_argparser() -> argparse.ArgumentParser:
         required=True,
         help="end time (ISO, YYYY-MM-DDTHH:MM:SS)",
     )
-    parser.add_argument("-r", "--repo", dest="repo", default="/repo/embargo", required=True, help="Butler repo")
+    parser.add_argument(
+        "-r",
+        "--repo",
+        dest="repo",
+        default="/repo/embargo",
+        required=True,
+        help="Butler repo",
+    )
     parser.add_argument(
         "-d",
         "--db",
@@ -202,7 +222,12 @@ def build_argparser() -> argparse.ArgumentParser:
         help="Consolidated Database connection string",
     )
     parser.add_argument(
-        "-E", "--efd", dest="efd_conn_str", default="usdf_efd", required=True, help="EFD connection string"
+        "-E",
+        "--efd",
+        dest="efd_conn_str",
+        default="usdf_efd",
+        required=True,
+        help="EFD connection string",
     )
     return parser
 
@@ -222,7 +247,7 @@ async def main() -> None:
     print(f"EFD: {efd}")
 
     config = read_config(args.config_name)
-    # print(f"Configs: {config}")   
+    # print(f"Configs: {config}")
 
     await process_interval(
         butler, db, efd, config, args.instrument, args.start_time, args.end_time
@@ -234,4 +259,3 @@ async def main() -> None:
     # Exemplo de execução
     # python transform_efd.py -i LATISS -s 2024-01-01T4:00:00  -e 2024-01-05T05:00:00 -r /repo/embargo -d sqlite://test.db -E usdf_efd -c test.yaml
     asyncio.run(main())
-
diff --git a/run.sh b/run.sh
@@ -1 +1 @@
-python python/lsst/consdb/transform_efd.py -i LATISS -s 2024-04-25T00:00:00  -e 2024-04-30T23:59:59 -r /repo/embargo -d sqlite://$PWD/.tmp/test.db -E usdf_efd -c $PWD/.tmp/config.yaml
+python python/lsst/consdb/transform_efd.py -i LATISS -s 2024-04-25T00:00:00  -e 2024-04-30T23:59:59 -r /repo/embargo -d sqlite://$PWD/.tmp/test.db -E usdf_efd -c $PWD/.tmp/config.yaml
Original file line number	Diff line number	Diff line change
Expand Up		@@ -5,4 +5,3 @@ COPY src/server.py /consdb-server/
		# Environment variables that must be set:
		# POSTGRES_URL
		ENTRYPOINT [ "gunicorn", "-b", "0.0.0.0:8000", "-w", "2", "server:app" ]
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		python python/lsst/consdb/transform_efd.py -i LATISS -s 2024-04-25T00:00:00 -e 2024-04-30T23:59:59 -r /repo/embargo -d sqlite://$PWD/.tmp/test.db -E usdf_efd -c $PWD/.tmp/config.yaml
		python python/lsst/consdb/transform_efd.py -i LATISS -s 2024-04-25T00:00:00 -e 2024-04-30T23:59:59 -r /repo/embargo -d sqlite://$PWD/.tmp/test.db -E usdf_efd -c $PWD/.tmp/config.yaml