ENH: Unify data handling under EpicsData

docs/source/upcoming_release_notes/73-enh_unify_data.rst

@@ -0,0 +1,22 @@
+73 enh_unify_data
+#################
+
+API Breaks
+----------
+- N/A
+
+Features
+--------
+- Standardize data handling to create and expect EpicsData instead of backend-specific data types.
+
+Bugfixes
+--------
+- N/A
+
+Maintenance
+-----------
+- Adjust Status, Severity enums to start at 0, matching EPICS
+
+Contributors
+------------
+- tangkong

superscore/client.py

@@ -8,12 +8,11 @@
 
 from superscore.backends import get_backend
 from superscore.backends.core import _Backend
-from superscore.control_layers import ControlLayer
+from superscore.control_layers import ControlLayer, EpicsData
 from superscore.control_layers.status import TaskStatus
 from superscore.errors import CommunicationError
 from superscore.model import (Collection, Entry, Nestable, Parameter, Readback,
                               Setpoint, Snapshot)
-from superscore.type_hints import AnyEpicsType
 from superscore.utils import build_abs_path
 
 logger = logging.getLogger(__name__)
@@ -39,7 +38,7 @@ def __init__(
         self.cl = control_layer
 
     @classmethod
-    def from_config(cls, cfg: Path = None):
+    def from_config(cls, cfg: Optional[Path] = None):
         """
         Create a client from the configuration file specification.
 
@@ -290,7 +289,7 @@ def _gather_data(
     def _build_snapshot(
         self,
         coll: Collection,
-        values: Dict[str, AnyEpicsType],
+        values: Dict[str, EpicsData],
     ) -> Snapshot:
         """
         Traverse a Collection, assembling a Snapshot using pre-fetched data
@@ -300,7 +299,7 @@ def _build_snapshot(
         ----------
         coll : Collection
             The collection being saved
-        values : Dict[str, AnyEpicsType]
+        values : Dict[str, EpicsData]
             A dictionary mapping PV names to pre-fetched values
 
         Returns
@@ -319,33 +318,50 @@ def _build_snapshot(
                 child = self.backend.get(child)
             if isinstance(child, Parameter):
                 if child.readback is not None:
+                    edata = self._value_or_default(
+                        values.get(child.readback.pv_name, None)
+                    )
                     readback = Readback(
                         pv_name=child.readback.pv_name,
                         description=child.readback.description,
-                        data=values[child.readback.pv_name]
+                        data=edata.data,
+                        status=edata.status,
+                        severity=edata.severity
                     )
                 else:
                     readback = None
+                edata = self._value_or_default(values.get(child.pv_name, None))
                 setpoint = Setpoint(
                     pv_name=child.pv_name,
                     description=child.description,
-                    data=values[child.pv_name],
+                    data=edata.data,
+                    status=edata.status,
+                    severity=edata.severity,
                     readback=readback
                 )
                 snapshot.children.append(setpoint)
             elif isinstance(child, Collection):
-                snapshot.append(self._build_snapshot(child, values))
+                snapshot.children.append(self._build_snapshot(child, values))
 
         snapshot.meta_pvs = []
         for pv in Collection.meta_pvs:
+            edata = self._value_or_default(values.get(pv, None))
             readback = Readback(
-                pv_name=readback.pv_name,
-                data=values[readback.pv_name]
+                pv_name=pv,
+                data=edata.data,
+                status=edata.status,
+                severity=edata.severity,
             )
             snapshot.meta_pvs.append(readback)
 
         return snapshot
 
+    def _value_or_default(self, value: Any) -> EpicsData:
+        """small helper for ensuring value is an EpicsData instance"""
+        if value is None or not isinstance(value, EpicsData):
+            return EpicsData(data=None)
+        return value
+
     def validate(self, entry: Entry):
         """
         Validate ``entry`` is properly formed and able to be inserted into

superscore/control_layers/__init__.py

@@ -1,2 +1,3 @@
+from ._base_shim import EpicsData  # noqa
 from .core import ControlLayer  # noqa
 from .status import TaskStatus  # noqa

superscore/control_layers/_aioca.py

@@ -5,16 +5,19 @@
 from typing import Any, Callable
 
 from aioca import CANothing, caget, camonitor, caput
+from aioca.types import AugmentedValue
+from epicscorelibs.ca import dbr
 
-from superscore.control_layers._base_shim import _BaseShim
+from superscore.control_layers._base_shim import EpicsData, _BaseShim
 from superscore.errors import CommunicationError
+from superscore.model import Severity, Status
 
 logger = logging.getLogger(__name__)
 
 
 class AiocaShim(_BaseShim):
     """async compatible EPICS channel access shim layer"""
-    async def get(self, address: str) -> Any:
+    async def get(self, address: str) -> EpicsData:
         """
         Get the value at the PV: ``address``.
 
@@ -25,7 +28,7 @@ async def get(self, address: str) -> Any:
 
         Returns
         -------
-        Any
+        EpicsData
             The data at ``address``.
 
         Raises
@@ -34,11 +37,13 @@ async def get(self, address: str) -> Any:
             If the caget operation fails for any reason.
         """
         try:
-            return await caget(address)
+            value = await caget(address, format=dbr.FORMAT_TIME)
         except CANothing as ex:
             logger.debug(f"CA get failed {ex.__repr__()}")
             raise CommunicationError(f'CA get failed for {ex}')
 
+        return self.value_to_epics_data(value)
+
     async def put(self, address: str, value: Any) -> None:
         """
         Put ``value`` to the PV ``address``.
@@ -73,3 +78,32 @@ def monitor(self, address: str, callback: Callable) -> None:
             The callback to run on updates to ``address``
         """
         camonitor(address, callback)
+
+    @staticmethod
+    def value_to_epics_data(value: AugmentedValue) -> EpicsData:
+        """
+        Creates an EpicsData instance from an aioca provided AugmentedValue
+        Assumes the augmented value was collected with FORMAT_TIME qualifier.
+        AugmentedValue subclasses primitive datatypes, so they can be used as
+        data directly.
+
+        Parameters
+        ----------
+        value : AugmentedValue
+            The value to repackage
+
+        Returns
+        -------
+        EpicsData
+            The filled EpicsData instance
+        """
+        severity = Severity(value.severity)
+        status = Status(value.status)
+        timestamp = value.timestamp
+
+        return EpicsData(
+            data=value,
+            status=status,
+            severity=severity,
+            timestamp=timestamp
+        )

superscore/control_layers/_base_shim.py

@@ -1,17 +1,32 @@
 """
 Base shim abstract base class
 """
-from typing import Any, Callable
+from __future__ import annotations
 
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Callable, Optional
+
+from superscore.model import Severity, Status
 from superscore.type_hints import AnyEpicsType
+from superscore.utils import utcnow
 
 
 class _BaseShim:
-    async def get(self, address: str) -> AnyEpicsType:
+    async def get(self, address: str) -> EpicsData:
         raise NotImplementedError
 
     async def put(self, address: str, value: Any):
         raise NotImplementedError
 
     def monitor(self, address: str, callback: Callable):
         raise NotImplementedError
+
+
+@dataclass
+class EpicsData:
+    """Unified EPICS data type for holding data and relevant metadata"""
+    data: Optional[AnyEpicsType]
+    status: Severity = Status.UDF
+    severity: Status = Severity.INVALID
+    timestamp: datetime = field(default_factory=utcnow)

superscore/control_layers/core.py

@@ -8,6 +8,7 @@
 from functools import singledispatchmethod
 from typing import Any, Callable, Dict, List, Optional, Union
 
+from superscore.control_layers._base_shim import EpicsData
 from superscore.control_layers.status import TaskStatus
 
 from ._aioca import AiocaShim
@@ -74,7 +75,7 @@ def shim_from_pv(self, address: str) -> _BaseShim:
         return shim
 
     @singledispatchmethod
-    def get(self, address: Union[str, list[str]]) -> Any:
+    def get(self, address: Union[str, Iterable[str]]) -> Union[EpicsData, Iterable[EpicsData]]:
         """
         Get the value(s) in ``address``.
         If a single pv is provided, will return a single value.
@@ -87,20 +88,20 @@ def get(self, address: Union[str, list[str]]) -> Any:
 
         Returns
         -------
-        Any
+        Union[EpicsData, list[EpicsData]]
             The requested data
         """
         # Dispatches to _get_single and _get_list depending on type
         print(f"PV is of an unsupported type: {type(address)}. Provide either "
               "a string or list of strings")
 
     @get.register
-    def _get_single(self, address: str) -> Any:
+    def _get_single(self, address: str) -> EpicsData:
         """Synchronously get a single ``address``"""
         return asyncio.run(self._get_one(address))
 
     @get.register
-    def _get_list(self, address: Iterable) -> Any:
+    def _get_list(self, address: Iterable) -> Iterable[EpicsData]:
         """Synchronously get a list of ``address``"""
         async def gathered_coros():
             coros = []

superscore/model.py

@@ -20,14 +20,14 @@
 
 
 class Severity(IntEnum):
-    NO_ALARM = auto()
+    NO_ALARM = 0
     MINOR = auto()
     MAJOR = auto()
     INVALID = auto()
 
 
 class Status(IntEnum):
-    NO_ALARM = auto()
+    NO_ALARM = 0
     READ = auto()
     WRITE = auto()
     HIHI = auto()

superscore/tests/db/filestore.json

@@ -20,8 +20,8 @@
         "creation_time": "2024-05-10T16:49:34.574875+00:00",
         "pv_name": "MY:MOTOR:mtr1.ACCL",
         "data": 2,
-        "status": 18,
-        "severity": 4,
+        "status": 17,
+        "severity": 3,
         "readback": null
       }
     },
@@ -80,26 +80,35 @@
             "creation_time": "2024-05-10T16:49:34.574951+00:00",
             "pv_name": "MY:PREFIX:mtr1.ACCL",
             "data": 2,
-            "status": 18,
-            "severity": 4
+            "status": 17,
+            "severity": 3,
+            "abs_tolerance": null,
+            "rel_tolerance": null,
+            "timeout": null
           },
           {
             "uuid": "8e380e15-5489-41db-a8a7-bc47a731f099",
             "description": "",
             "creation_time": "2024-05-10T16:49:34.574987+00:00",
             "pv_name": "MY:PREFIX:mtr1.VELO",
             "data": 2,
-            "status": 18,
-            "severity": 4
+            "status": 17,
+            "severity": 3,
+            "abs_tolerance": null,
+            "rel_tolerance": null,
+            "timeout": null
           },
           {
             "uuid": "ed8318d7-8c72-4d47-82d0-d75216d11565",
             "description": "",
             "creation_time": "2024-05-10T16:49:34.575022+00:00",
             "pv_name": "MY:PREFIX:mtr1.PREC",
             "data": 6,
-            "status": 18,
-            "severity": 4
+            "status": 17,
+            "severity": 3,
+            "abs_tolerance": null,
+            "rel_tolerance": null,
+            "timeout": null
           }
         ],
         "tags": [],

superscore/tests/test_client.py

@@ -6,6 +6,7 @@
 
 from superscore.backends.filestore import FilestoreBackend
 from superscore.client import Client
+from superscore.control_layers import EpicsData
 from superscore.errors import CommunicationError
 from superscore.model import Parameter, Readback, Root, Setpoint
 
@@ -84,7 +85,7 @@ def test_snap(
     coll = sample_database.entries[2]
     coll.children.append(parameter_with_readback)
 
-    get_mock.side_effect = range(5)
+    get_mock.side_effect = [EpicsData(i) for i in range(5)]
     snapshot = mock_client.snap(coll)
     assert get_mock.call_count == 5
     assert all([snapshot.children[i].data == i for i in range(4)])  # children saved in order
@@ -97,7 +98,8 @@ def test_snap(
 @patch('superscore.control_layers.core.ControlLayer._get_one')
 def test_snap_exception(get_mock, mock_client: Client, sample_database: Root):
     coll = sample_database.entries[2]
-    get_mock.side_effect = [0, 1, CommunicationError, 3, 4]
+    get_mock.side_effect = [EpicsData(0), EpicsData(1), CommunicationError,
+                            EpicsData(3), EpicsData(4)]
     snapshot = mock_client.snap(coll)
     assert snapshot.children[2].data is None