Merge branch 'develop'

* develop:
  Uncomment master deployment pipeline stage
  Rename LSSTTS_DEV_VERSION for dev_cycle
  Rollback to develop-env c0017.000
  Remove deprecated command
  Upgrade to lsstts/develop-env:c0018.000
  Build docker images from tickets branch
  Remove except specification to catch all exceptions
  Improve EFD tests
  Add EFD reconnection when after it failst on startup
  Add flake8 project config file
  Linter fixes

.env

@@ -2,3 +2,8 @@
 # Cycle to build
 #
 cycle=c0014
+
+#
+# Development cycle to build
+#
+dev_cycle=c0017.000

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+ignore = E128, E231
+max-line-length = 120

Dockerfile

@@ -1,5 +1,5 @@
-ARG LSSTTS_DEV_VERSION=develop
-FROM lsstts/develop-env:${LSSTTS_DEV_VERSION}
+ARG dev_cycle=develop
+FROM lsstts/develop-env:${dev_cycle}
 
 WORKDIR /usr/src/love/
 

Dockerfile-dev

@@ -1,5 +1,5 @@
-ARG LSSTTS_DEV_VERSION=develop
-FROM lsstts/develop-env:${LSSTTS_DEV_VERSION}
+ARG dev_cycle=develop
+FROM lsstts/develop-env:${dev_cycle}
 
 WORKDIR /usr/src/love/commander
 COPY requirements-dev.txt .

Jenkinsfile

@@ -4,7 +4,7 @@ pipeline {
     registryCredential = "dockerhub-inriachile"
     dockerImageName = "lsstts/love-commander:"
     dockerImage = ""
-    LSSTTS_DEV_VERSION = "c0016.001"
+    dev_cycle = "c0017.000"
     user_ci = credentials('lsst-io')
     LTD_USERNAME="${user_ci_USR}"
     LTD_PASSWORD="${user_ci_PSW}"
@@ -19,6 +19,7 @@ pipeline {
           branch "bugfix/*"
           branch "hotfix/*"
           branch "release/*"
+          branch "tickets/*"
         }
       }
       steps {
@@ -29,13 +30,13 @@ pipeline {
           if (slashPosition > 0) {
             git_tag = git_branch.substring(slashPosition + 1, git_branch.length())
             git_branch = git_branch.substring(0, slashPosition)
-            if (git_branch == "release" || git_branch == "hotfix" || git_branch == "bugfix") {
+            if (git_branch == "release" || git_branch == "hotfix" || git_branch == "bugfix" || git_branch == "tickets") {
               image_tag = git_tag
             }
           }
           dockerImageName = dockerImageName + image_tag
           echo "dockerImageName: ${dockerImageName}"
-          dockerImage = docker.build(dockerImageName, "--build-arg LSSTTS_DEV_VERSION=${LSSTTS_DEV_VERSION} .")
+          dockerImage = docker.build(dockerImageName, "--build-arg dev_cycle=${dev_cycle} .")
         }
       }
     }
@@ -48,6 +49,7 @@ pipeline {
           branch "bugfix/*"
           branch "hotfix/*"
           branch "release/*"
+          branch "tickets/*"
         }
       }
       steps {
@@ -106,13 +108,14 @@ pipeline {
         build(job: '../LOVE-integration-tools/develop', wait: false)
       }
     }
-    // stage("Trigger master deployment") {
-    //   when {
-    //     branch "master"
-    //   }
-    //   steps {
-    //     build(job: '../LOVE-integration-tools/master', wait: false)
-    //   }
-    // }
+
+    stage("Trigger master deployment") {
+      when {
+        branch "master"
+      }
+      steps {
+        build(job: '../LOVE-integration-tools/master', wait: false)
+      }
+    }
   }
 }

commander/efd.py

@@ -1,14 +1,13 @@
 """Define the Heartbeats subapplication, which provides the endpoints to request a heartbeat."""
 from aiohttp import web
-import json
-from datetime import datetime
 import lsst_efd_client
 from astropy.time import Time, TimeDelta
-import pytest
 import asyncio
-from astropy.time import Time, TimeDelta
 import os
 
+efd_client = None
+
+
 def create_app():
     """Create the EFD application
 
@@ -19,18 +18,38 @@ def create_app():
     """
     efd_app = web.Application()
     efd_instance = os.environ.get("EFD_INSTANCE", "summit_efd")
-    efd_client = lsst_efd_client.EfdClient(efd_instance)
+
+    def connect_to_efd_intance():
+        global efd_client
+        try:
+            efd_client = lsst_efd_client.EfdClient(efd_instance)
+        except Exception:
+            efd_client = None
+
+    connect_to_efd_intance()
+
+    def unavailableEfdClient():
+        return web.json_response(
+            {"ack": f"EFD Client could not stablish connection"}, status=400
+        )
 
     async def query_efd_timeseries(request):
+        global efd_client
+        if efd_client is None:
+            connect_to_efd_intance()
+
+        if efd_client is None:
+            return unavailableEfdClient()
+
         req = await request.json()
 
         start_date = req["start_date"]
         time_window = int(req["time_window"])
         cscs = req["cscs"]
         resample = req["resample"]
-        
+
         parsed_date = Time(start_date, scale="tai")
-        time_delta = TimeDelta(time_window*60, format="sec", scale="tai")
+        time_delta = TimeDelta(time_window * 60, format="sec", scale="tai")
         query_tasks = []
         sources = []
         for csc in cscs:
@@ -40,25 +59,28 @@ async def query_efd_timeseries(request):
                 for topic in topics:
                     fields = topics[topic]
                     task = efd_client.select_time_series(
-                        f"lsst.sal.{csc}.{topic}", 
+                        f"lsst.sal.{csc}.{topic}",
                         fields,
                         parsed_date,
                         time_delta,
                         index=int(index),
-                        is_window=True
+                        is_window=True,
                     )
                     sources.append(f"{csc}-{index}-{topic}")
                     query_tasks.append(task)
 
-        results = [r.resample(resample).mean() if not r.empty else r for r in await asyncio.gather(*query_tasks)]
+        results = [
+            r.resample(resample).mean() if not r.empty else r
+            for r in await asyncio.gather(*query_tasks)
+        ]
         results = [r.to_dict() for r in results]
 
         for res in results:
             for field in res:
                 items = res[field].items()
                 res[field] = [{"ts": str(item[0]), "value": item[1]} for item in items]
 
-        response_data = dict(zip(sources,  results))
+        response_data = dict(zip(sources, results))
         return web.json_response(response_data)
 
     efd_app.router.add_post("/timeseries", query_efd_timeseries)

commander/start-daemon-dev.sh

@@ -3,7 +3,7 @@
 
 
 export PYTHONPATH=$PYTHONPATH:/usr/src/love/commander
-/home/saluser/repos/ts_sal/bin/make_idl_files.py LOVE
 
 cd /usr/src/love
-adev runserver commander -p 5000
+
+adev runserver commander -p 5000

docker-compose.yml

@@ -7,7 +7,7 @@ services:
       context: .
       dockerfile: Dockerfile-dev
       args:
-        LSSTTS_DEV_VERSION: c0017.000
+        dev_cycle: ${dev_cycle}
     image: love-commander-image-mount
     volumes:
       - .:/usr/src/love

tests/test_efd.py

@@ -1,78 +1,104 @@
-import json
 import asyncio
-from aiohttp import web
-from unittest.mock import patch
-from itertools import chain, combinations
-from lsst.ts import salobj
 from commander.app import create_app
-from utils import NumpyEncoder
-from tests import conftest
-import pytest
 import pandas as pd
-from unittest.mock import patch, call, MagicMock
+from unittest.mock import patch, MagicMock
+from aiohttp.test_utils import TestClient, TestServer
+
 
 # Patch for using MagicMock in async environments
 async def async_magic():
     pass
+
+
 MagicMock.__await__ = lambda x: async_magic().__await__()
 
+
 class MockEFDClient(object):
-    async def select_time_series(cls, topic_name, fields, start, end, is_window=False, index=None):
+    async def select_time_series(
+        cls, topic_name, fields, start, end, is_window=False, index=None
+    ):
         f = asyncio.Future()
         data = {}
         for field in fields:
             data[field] = {
-                pd.Timestamp("2020-03-06 21:49:41"):0.21,
-                pd.Timestamp("2020-03-06 21:50:41"):0.21,
-                pd.Timestamp("2020-03-06 21:51:41"):0.21,
-                pd.Timestamp("2020-03-06 21:52:41"):0.21,
-                pd.Timestamp("2020-03-06 21:53:41"):0.21
+                pd.Timestamp("2020-03-06 21:49:41"): 0.21,
+                pd.Timestamp("2020-03-06 21:50:41"): 0.21,
+                pd.Timestamp("2020-03-06 21:51:41"): 0.21,
+                pd.Timestamp("2020-03-06 21:52:41"): 0.21,
+                pd.Timestamp("2020-03-06 21:53:41"): 0.21,
             }
 
         df = pd.DataFrame.from_dict(data)
         f.set_result(df)
         return df
 
-# Start patching `efd_client`.
-mock_efd_patcher = patch("lsst_efd_client.EfdClient")
-mock_efd_client = mock_efd_patcher.start()
-mock_efd_client.return_value = MockEFDClient()
+
+# For EFD client's timeouts
+def raise_exception(name):
+    print(name)
+    raise ConnectionError
+
 
 async def test_efd_timeseries(client):
     """ Test the get timeseries response."""
+    # Start patching `efd_client`.
+    mock_efd_patcher = patch("lsst_efd_client.EfdClient")
+    mock_efd_client = mock_efd_patcher.start()
+    mock_efd_client.return_value = MockEFDClient()
+    loop = asyncio.get_event_loop()
+    app = await create_app()
+    async with TestClient(TestServer(app), loop=loop) as client:
 
-
-    cscs = {
-        "ATDome": {
-            0: {
-                "topic1": ["field1"]
-            },
-        },
-        "ATMCS": {
-            1: {
-                "topic2": ["field2", "field3"]
-            },
+        cscs = {
+            "ATDome": {0: {"topic1": ["field1"]},},
+            "ATMCS": {1: {"topic2": ["field2", "field3"]},},
         }
-    }
-    request_data = {
-        "start_date": "2020-03-16T12:00:00",
-        "time_window": 15,
-        "cscs": cscs,
-        "resample": "1min",
-    }
-    response = await client.post("/efd/timeseries", json=request_data)
-    assert response.status == 200
-
-    response_data = await response.json()
-    assert "ATDome-0-topic1" in list(response_data.keys())
-    assert "ATMCS-1-topic2" in list(response_data.keys())
-    assert len(response_data["ATDome-0-topic1"]) == 1
-    assert len(response_data["ATMCS-1-topic2"]) == 2
-    # Endpoint truncates seconds due to resample
-    assert response_data["ATDome-0-topic1"]["field1"][0]["ts"] == "2020-03-06 21:49:00"
-    assert response_data["ATDome-0-topic1"]["field1"][0]["value"] == 0.21
-
-    # Stop patching `efd_client`.
-    mock_efd_patcher.stop()
+        request_data = {
+            "start_date": "2020-03-16T12:00:00",
+            "time_window": 15,
+            "cscs": cscs,
+            "resample": "1min",
+        }
+        response = await client.post("/efd/timeseries", json=request_data)
+        assert response.status == 200
+
+        response_data = await response.json()
+        assert "ATDome-0-topic1" in list(response_data.keys())
+        assert "ATMCS-1-topic2" in list(response_data.keys())
+        assert len(response_data["ATDome-0-topic1"]) == 1
+        assert len(response_data["ATMCS-1-topic2"]) == 2
+        # Endpoint truncates seconds due to resample
+        assert (
+            response_data["ATDome-0-topic1"]["field1"][0]["ts"] == "2020-03-06 21:49:00"
+        )
+        assert response_data["ATDome-0-topic1"]["field1"][0]["value"] == 0.21
+
+        # Stop patching `efd_client`.
+        mock_efd_patcher.stop()
 
 
+async def test_efd_timeseries_with_errors():
+    """ Test the get timeseries response with errors."""
+    # Start patching `efd_client`.
+    mock_efd_patcher = patch("lsst_efd_client.EfdClient")
+    mock_efd_client = mock_efd_patcher.start()
+    mock_efd_client.return_value = MockEFDClient()
+    mock_efd_client.side_effect = raise_exception
+    loop = asyncio.get_event_loop()
+    app = await create_app()
+    async with TestClient(TestServer(app), loop=loop) as client:
+        cscs = {
+            "ATDome": {0: {"topic1": ["field1"]},},
+            "ATMCS": {1: {"topic2": ["field2", "field3"]},},
+        }
+        request_data = {
+            "start_date": "2020-03-16T12:00:00",
+            "time_window": 15,
+            "cscs": cscs,
+            "resample": "1min",
+        }
+        response = await client.post("/efd/timeseries", json=request_data)
+        assert response.status == 400
+
+        # Stop patching `efd_client`.
+        mock_efd_patcher.stop()