Merge pull request #22 from lsst-ts/release/3.0.0

Release/3.0.0

.env

@@ -0,0 +1,4 @@
+#
+# Cycle to build
+#
+cycle=c0014

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: check-yaml
+        exclude: conda/meta.yaml
+
+  - repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+      - id: black
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.9
+    hooks:
+      - id: flake8

Dockerfile

@@ -1,12 +1,14 @@
-FROM lsstts/develop-env:b101
+ARG LSSTTS_DEV_VERSION=develop
+FROM lsstts/develop-env:${LSSTTS_DEV_VERSION}
 
 WORKDIR /usr/src/love/
 
 COPY . .
 # Missing SAL topics
 RUN source /opt/lsst/software/stack/loadLSST.bash \
-    && source /home/saluser/repos/ts_sal/setup.env \
+    && source /home/saluser/.setup_salobj.sh \
     && setup ts_sal -t current \
+    && pip install lsst_efd_client \
     && /home/saluser/repos/ts_sal/bin/make_idl_files.py Watcher
 
 WORKDIR /home/saluser

Dockerfile-deploy

@@ -0,0 +1,20 @@
+ARG cycle
+
+FROM ts-dockerhub.lsst.org/deploy-env:${cycle}
+
+WORKDIR /usr/src/love/
+
+COPY . .
+
+ARG idl
+RUN source /home/saluser/.setup_sal_env.sh && \
+    pip install kafkit[aiohttp] aiokafka lsst_efd_client && \
+    pip install -r requirements-dev.txt
+
+COPY commander/start-daemon-deploy.sh /home/saluser/.startup.sh
+USER root
+RUN chown saluser:saluser /home/saluser/.startup.sh && \
+    chmod a+x /home/saluser/.startup.sh
+USER saluser
+
+WORKDIR /home/saluser

Dockerfile-dev

@@ -1,14 +1,17 @@
-FROM lsstts/develop-env:b101
+ARG LSSTTS_DEV_VERSION=develop
+FROM lsstts/develop-env:${LSSTTS_DEV_VERSION}
 
 WORKDIR /usr/src/love/commander
 COPY requirements-dev.txt .
 
 # LOVE requirements
-RUN source /opt/lsst/software/stack/loadLSST.bash && pip install -r requirements-dev.txt
+RUN source /opt/lsst/software/stack/loadLSST.bash && \
+    pip install kafkit[aiohttp] aiokafka lsst_efd_client && \
+    pip install -r requirements-dev.txt
 
 # Missing SAL topics
 RUN source /opt/lsst/software/stack/loadLSST.bash \
-    && source /home/saluser/repos/ts_sal/setup.env \
+    && source /home/saluser/.setup_salobj.sh \
     && setup ts_sal -t current \
     && /home/saluser/repos/ts_sal/bin/make_idl_files.py Watcher
 

Jenkinsfile

@@ -4,6 +4,10 @@ pipeline {
     registryCredential = "dockerhub-inriachile"
     dockerImageName = "lsstts/love-commander:"
     dockerImage = ""
+    LSSTTS_DEV_VERSION = "c0016.001"
+    user_ci = credentials('lsst-io')
+    LTD_USERNAME="${user_ci_USR}"
+    LTD_PASSWORD="${user_ci_PSW}"
   }
 
   stages {
@@ -31,7 +35,7 @@ pipeline {
           }
           dockerImageName = dockerImageName + image_tag
           echo "dockerImageName: ${dockerImageName}"
-          dockerImage = docker.build dockerImageName
+          dockerImage = docker.build(dockerImageName, "--build-arg LSSTTS_DEV_VERSION=${LSSTTS_DEV_VERSION} .")
         }
       }
     }
@@ -69,6 +73,31 @@ pipeline {
     //   }
     // }
 
+    stage("Deploy documentation") {
+      agent {
+        docker {
+          alwaysPull true
+          image 'lsstts/develop-env:develop'
+          args "-u root --entrypoint=''"
+        }
+      }
+      when {
+        anyOf {
+          changeset "docs/*"
+        }
+      }
+      steps {
+        script {
+          sh "pwd"
+          sh """
+            source /home/saluser/.setup_dev.sh
+            pip install ltd-conveyor
+            ltd upload --product love-commander --git-ref ${GIT_BRANCH} --dir ./docs
+          """
+        }
+      }
+    }
+
     stage("Trigger develop deployment") {
       when {
         branch "develop"

README.md

@@ -65,3 +65,18 @@ Once inside the container and in the `love` folder you can build the documentati
 cd docsrc
 ./create_docs.sh
 ```
+
+### Linting & Formatting
+In order to maintaing code linting and formatting we use `pre-commit` that runs **Flake8** (https://flake8.pycqa.org/) and **Black** (https://github.com/psf/black) using Git Hooks. To enable this you have to:
+
+1. Install `pre-commit` in your local development environment:
+```
+pip install pre-commit
+```
+
+2. Set up the git hook scripts running:
+```
+pre-commit install
+```
+
+3. Start developing! Linter and Formatter will be executed on every commit you make

commander/app.py

@@ -5,6 +5,8 @@
 from .commands import create_app as create_cmd_app
 from .heartbeats import create_app as create_heartbeat_app
 from .salinfo import create_app as create_salinfo_app
+from .lovecsc import create_app as create_lovecsc_app
+from .efd import create_app as create_efd_app
 
 
 async def create_app(*args, **kwargs):
@@ -19,6 +21,9 @@ async def create_app(*args, **kwargs):
 
     app.add_subapp("/cmd/", create_cmd_app())
     app.add_subapp("/heartbeat/", create_heartbeat_app())
+    app.add_subapp("/lovecsc/", create_lovecsc_app())
+    app.add_subapp("/efd/", create_efd_app())
+
     app.add_subapp(
         "/salinfo/",
         await create_salinfo_app(remotes_len_limit=kwargs.get("remotes_len_limit")),

commander/commands.py

@@ -12,11 +12,13 @@ def create_app():
     object
         The application instance
     """
+    domain = None
     remotes = {}
 
     cmd = web.Application()
 
     async def start_cmd(request):
+        nonlocal domain
         data = await request.json()
         try:
             assert "csc" in data
@@ -26,7 +28,8 @@ async def start_cmd(request):
         except AssertionError:
             return web.json_response(
                 {
-                    "ack": f"Request must have JSON data with the following keys: csc, salindex, cmd_name, params. Received {json.dumps(data)}"
+                    "ack": f"Request must have JSON data with the following "
+                    f"keys: csc, salindex, cmd_name, params. Received {json.dumps(data)}"
                 },
                 status=400,
             )
@@ -36,18 +39,34 @@ async def start_cmd(request):
         cmd_name = data["cmd"]
         params = data["params"]
         remote_name = f"{csc}.{salindex}"
+
+        # Only create domain if it does not already exist.
+        if domain is None:
+            print("Creating salobj.Domain()")
+            domain = salobj.Domain()
+            domain.default_identity = "LOVE"
+
+        # Only create remote if it does not exist already.
         if remote_name not in remotes:
-            remotes[remote_name] = salobj.Remote(salobj.Domain(), csc, salindex)
+            print(f"Creating remote {remote_name}.")
+            # Create remote for commanding only, exclude all events and
+            # telemetry topics
+            remotes[remote_name] = salobj.Remote(domain, csc, salindex, include=[])
             await remotes[remote_name].start_task
 
         cmd = getattr(remotes[remote_name], cmd_name)
         cmd.set(**params)
 
         try:
-            cmd_result = await cmd.start(timeout=10)
+            cmd_result = await cmd.start(timeout=5)
             return web.json_response({"ack": cmd_result.result})
-        except salobj.AckTimeoutError:
-            return web.json_response({"ack": "Command time out"}, status=504)
+        except salobj.AckTimeoutError as e:
+            msg = (
+                "No ack received from component."
+                if e.ackcmd == salobj.SalRetCode.CMD_NOACK
+                else f"Last ack received {e.ackcmd}."
+            )
+            return web.json_response({"ack": f"Command time out. {msg}"}, status=504)
 
     cmd.router.add_post("/", start_cmd)
 

commander/efd.py

@@ -0,0 +1,73 @@
+"""Define the Heartbeats subapplication, which provides the endpoints to request a heartbeat."""
+from aiohttp import web
+import json
+from datetime import datetime
+import lsst_efd_client
+from astropy.time import Time, TimeDelta
+import pytest
+import asyncio
+from astropy.time import Time, TimeDelta
+import os
+
+def create_app():
+    """Create the EFD application
+
+    Returns
+    -------
+    object
+        The application instance
+    """
+    efd_app = web.Application()
+    efd_instance = os.environ.get("EFD_INSTANCE", "summit_efd")
+    efd_client = lsst_efd_client.EfdClient(efd_instance)
+
+    async def query_efd_timeseries(request):
+        req = await request.json()
+
+        start_date = req["start_date"]
+        time_window = int(req["time_window"])
+        cscs = req["cscs"]
+        resample = req["resample"]
+
+        parsed_date = Time(start_date, scale="tai")
+        time_delta = TimeDelta(time_window*60, format="sec", scale="tai")
+        query_tasks = []
+        sources = []
+        for csc in cscs:
+            indices = cscs[csc]
+            for index in indices:
+                topics = indices[index]
+                for topic in topics:
+                    fields = topics[topic]
+                    task = efd_client.select_time_series(
+                        f"lsst.sal.{csc}.{topic}", 
+                        fields,
+                        parsed_date,
+                        time_delta,
+                        index=int(index),
+                        is_window=True
+                    )
+                    sources.append(f"{csc}-{index}-{topic}")
+                    query_tasks.append(task)
+
+        results = [r.resample(resample).mean() if not r.empty else r for r in await asyncio.gather(*query_tasks)]
+        results = [r.to_dict() for r in results]
+
+        for res in results:
+            for field in res:
+                items = res[field].items()
+                res[field] = [{"ts": str(item[0]), "value": item[1]} for item in items]
+
+        response_data = dict(zip(sources,  results))
+        return web.json_response(response_data)
+
+    efd_app.router.add_post("/timeseries", query_efd_timeseries)
+    efd_app.router.add_post("/timeseries/", query_efd_timeseries)
+
+    async def on_cleanup(efd_app):
+        # Do cleanup
+        pass
+
+    efd_app.on_cleanup.append(on_cleanup)
+
+    return efd_app

commander/heartbeats.py

@@ -17,7 +17,7 @@ def create_app():
     heartbeat = web.Application()
 
     async def start_heartbeat(request):
-        data = await request.read()
+        await request.read()
         now = datetime.now()
         timestamp = datetime.timestamp(now)
         return web.json_response({"timestamp": timestamp})

commander/lovecsc.py

@@ -0,0 +1,72 @@
+"""Define the SAL Info subapplication, which provides the endpoints to request info from SAL."""
+from aiohttp import web
+from lsst.ts import salobj
+import utils
+import json
+
+STD_TIMEOUT = 15  # timeout for command ack
+
+index_gen = salobj.index_generator()
+
+
+def create_app(*args, **kwargs):
+    """Create the LOVECsc application
+
+    Returns
+    -------
+    object
+        The application instance
+    """
+    lovecsc_app = web.Application()
+    csc = salobj.Controller("LOVE", index=None, do_callbacks=False)
+
+    async def post_observing_log(request):
+        """Handle post observing log requests.
+
+        Parameters
+        ----------
+        request : Request
+            The original HTTP request
+
+        Returns
+        -------
+        Response
+            The response for the HTTP request with the following structure:
+
+            .. code-block:: json
+
+                {
+                    "ack": "<Description about the success state of the request>"
+                }
+        """
+
+        data = await request.json()
+
+        try:
+            assert "user" in data
+            assert "message" in data
+        except AssertionError:
+            return web.json_response(
+                {
+                    "ack": f"Request must have JSON data with the following keys: user, message. Received {json.dumps(data)}"
+                },
+                status=400,
+            )
+
+        user = data["user"]
+        message = data["message"]
+
+        # LOVECsc
+        csc.evt_observingLog.set(user=user, message=message)
+        csc.evt_observingLog.put()
+
+        return web.json_response({"ack": "Added new observing log to SAL"}, status=200)
+
+    lovecsc_app.router.add_post("/observinglog", post_observing_log)
+
+    async def on_cleanup(lovecsc_app):
+        await csc.close()
+
+    lovecsc_app.on_cleanup.append(on_cleanup)
+
+    return lovecsc_app