From 4779eb6454c3ef355ace8967d86a009b22762969 Mon Sep 17 00:00:00 2001 From: Peter Wegmann <85115389+Bilchreis@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:34:23 +0100 Subject: [PATCH] Fix-array-shape (#18) * added dtype tests and max_depth warning * add struct of empty arrays test * remove print * add empty array tests, and remove sring len warning --------- Co-authored-by: Peter Braun --- src/secop_ophyd/SECoPSignal.py | 28 +++- src/secop_ophyd/util.py | 110 ++++++++------ tests/test_dtype.py | 264 ++++++++++++++++++++++----------- 3 files changed, 271 insertions(+), 131 deletions(-) diff --git a/src/secop_ophyd/SECoPSignal.py b/src/secop_ophyd/SECoPSignal.py index 0a82025..d185aa2 100644 --- a/src/secop_ophyd/SECoPSignal.py +++ b/src/secop_ophyd/SECoPSignal.py @@ -1,4 +1,5 @@ import asyncio +import warnings from functools import wraps from typing import Any, Callable, Dict, Optional @@ -31,6 +32,9 @@ ArrayOf, ) +# max depth for datatypes supported by tiled/databroker +MAX_DEPTH = 1 + class LocalBackend(SignalBackend): """Class for the 'argument' and 'result' Signal backends of a SECoP_CMD_Device. @@ -77,7 +81,7 @@ def __init__( self.describe_dict["source"] = self.source("") - self.describe_dict.update(self.SECoP_type_info.describe_dict) + self.describe_dict.update(self.SECoP_type_info.get_datakey()) for property_name, prop_val in self.datainfo.items(): if property_name == "type": @@ -242,6 +246,14 @@ def __init__(self, path: Path, secclient: AsyncFrappyClient) -> None: self.SECoP_type_info: SECoPdtype = SECoPdtype(self.SECoPdtype_obj) + if self.SECoP_type_info.max_depth > MAX_DEPTH: + warnings.warn( + f"The datatype of parameter '{path._accessible_name}' has a maximum" + f"depth of {self.SECoP_type_info.max_depth}. Tiled & Databroker only" + f"support a Depth upto {MAX_DEPTH}" + f"dtype_descr: {self.SECoP_type_info.dtype_descr}" + ) + self.describe_dict: dict = {} self.source_name = ( @@ -260,7 +272,7 @@ def __init__(self, path: Path, secclient: AsyncFrappyClient) -> None: self.describe_dict["source"] = self.source_name # add gathered keys from SECoPdtype: - self.describe_dict.update(self.SECoP_type_info.describe_dict) + self.describe_dict.update(self.SECoP_type_info.get_datakey()) for property_name, prop_val in self._param_description.items(): # skip datainfo (treated seperately) @@ -303,7 +315,7 @@ async def get_datakey(self, source: str) -> DataKey: # this ensures the datakey is updated to the latest cached value SECoPReading(entry=dataset, secop_dt=self.SECoP_type_info) - self.describe_dict.update(self.SECoP_type_info.describe_dict) + self.describe_dict.update(self.SECoP_type_info.get_datakey()) return self.describe_dict @@ -390,13 +402,21 @@ def __init__( self.SECoPdtype_obj: DataType = secop_dtype_obj_from_json(self._prop_value) self.SECoP_type_info: SECoPdtype = SECoPdtype(self.SECoPdtype_obj) + if self.SECoP_type_info.max_depth > MAX_DEPTH: + warnings.warn( + f"The datatype of parameter '{prop_key}' has a maximum" + f"depth of {self.SECoP_type_info.max_depth}. Tiled & Databroker only" + f"support a Depth upto {MAX_DEPTH}" + f"dtype_descr: {self.SECoP_type_info.dtype_descr}" + ) + # SECoP metadata is static and can only change when connection is reset self.describe_dict = {} self.source_name = prop_key self.describe_dict["source"] = self.source_name # add gathered keys from SECoPdtype: - self.describe_dict.update(self.SECoP_type_info.describe_dict) + self.describe_dict.update(self.SECoP_type_info.get_datakey()) self._secclient: AsyncFrappyClient = secclient # TODO full property path diff --git a/src/secop_ophyd/util.py b/src/secop_ophyd/util.py index 34f5f17..f8d0ed9 100644 --- a/src/secop_ophyd/util.py +++ b/src/secop_ophyd/util.py @@ -2,6 +2,7 @@ import copy import time +import warnings from abc import ABC, abstractmethod from functools import reduce from itertools import chain @@ -156,6 +157,7 @@ class DtypeNP(ABC): secop_dtype: DataType name: str | None array_element: bool = False + max_depth: int = 0 @abstractmethod def make_numpy_dtype(self) -> tuple: @@ -333,10 +335,10 @@ def __init__( self.array_element = array_element if string_dt.maxchars == 1 << 64: - Warning( - "maxchars was not set, default max char lenght is set to: " - + str(STR_LEN_DEFAULT) - ) + # warnings.warn( + # "maxchars was not set, default max char lenght is set to: " + # + str(STR_LEN_DEFAULT) + # ) self.strlen = STR_LEN_DEFAULT else: @@ -367,6 +369,9 @@ def __init__( for (name, member) in struct_dt.members.items() } + max_depth = [member.max_depth for member in self.members.values()] + self.max_depth = 1 + max(max_depth) + def make_numpy_dtype(self) -> tuple: dt_list = [] for member in self.members.values(): @@ -406,10 +411,15 @@ def __init__( self.secop_dtype = tuple_dt self.array_element = array_element self.members: list[DtypeNP] = [ - dt_factory(member, array_element=self.array_element) - for member in tuple_dt.members + dt_factory( + secop_dt=member, name="f" + str(idx), array_element=self.array_element + ) + for idx, member in enumerate(tuple_dt.members) ] + max_depth = [member.max_depth for member in self.members] + self.max_depth = 1 + max(max_depth) + def make_numpy_dtype(self) -> tuple: dt_list = [] for member in self.members: @@ -459,6 +469,8 @@ def __init__( self.members: DtypeNP = dt_factory(array_dt.members, array_element=True) self.root_type: DtypeNP + self.max_depth = self.members.max_depth + if isinstance(self.members, ArrayNP): self.shape.extend(self.members.shape) self.root_type = self.members.root_type @@ -473,11 +485,10 @@ def __init__( self.root_type = self.members if self.array_element and self.ragged: - pass - # raise NestedRaggedArray( - # "ragged arrays inside of arrays of copmposite datatypes (struct/tuple)" - # "are not supported" - # ) + warnings.warn( + "ragged arrays inside of arrays of copmposite datatypes (struct/tuple)" + "are not supported" + ) def make_numpy_dtype(self) -> tuple: if self.shape == []: @@ -487,20 +498,27 @@ def make_numpy_dtype(self) -> tuple: def make_concrete_numpy_dtype(self, value) -> tuple: - if self.shape == []: - return self.members.make_concrete_numpy_dtype(value) - elif self.ragged is False: + if self.ragged is False: return ( self.name, list(self.members.make_concrete_numpy_dtype(value)).pop(), self.shape, ) else: - return ( - self.name, - list(self.members.make_concrete_numpy_dtype(value)).pop(), - [len(value)], - ) + if value == []: + member_np = self.members.make_concrete_numpy_dtype(None) + val_shape = [0] + elif value is None: + member_np = self.members.make_concrete_numpy_dtype(None) + val_shape = [] + else: + member_np = self.members.make_concrete_numpy_dtype(value[0]) + val_shape = [len(value)] + + if isinstance(self.members, ArrayNP): + val_shape = val_shape + member_np[2] + + return (self.name, member_np[1], val_shape) def make_numpy_compatible_list(self, value: list): return [self.members.make_numpy_compatible_list(elem) for elem in value] @@ -540,10 +558,10 @@ def __init__(self, datatype: DataType) -> None: self._is_composite: bool = False - self.describe_dict: dict = {} - self.dtype_tree = dt_factory(datatype) + self.max_depth: int = self.dtype_tree.max_depth + if isinstance(self.dtype_tree, ArrayNP): self.shape = self.dtype_tree.shape self._is_composite = ( @@ -567,12 +585,8 @@ def __init__(self, datatype: DataType) -> None: # all composite Dtypes are transported as numpy arrays self.dtype = "array" - self.dtype_str = self.numpy_dtype.str - self.describe_dict["dtype_str"] = self.dtype_str - self.dtype_descr = self.numpy_dtype.descr - self.describe_dict["dtype_descr"] = self.dtype_descr # Scalar atomic Datatypes and arrays of atomic dataypes else: @@ -585,9 +599,18 @@ def __init__(self, datatype: DataType) -> None: else: self.dtype = SECOP2DTYPE[datatype.__class__] - self.describe_dict["dtype"] = self.dtype - self.describe_dict["shape"] = self.shape - self.describe_dict["SECOP_datainfo"] = self.secop_dtype_str + def get_datakey(self): + describe_dict: dict = {} + # Composite Datatypes & Arrays of COmposite Datatypes + if self._is_composite: + describe_dict["dtype_str"] = self.dtype_str + describe_dict["dtype_descr"] = self.dtype_descr + + describe_dict["dtype"] = self.dtype + describe_dict["shape"] = self.shape + describe_dict["SECOP_datainfo"] = self.secop_dtype_str + + return describe_dict def _secop2numpy_array(self, value) -> np.ndarray: np_list = self.dtype_tree.make_numpy_compatible_list(value) @@ -611,28 +634,29 @@ def val2secop(self, input_val) -> Any: return self.raw_dtype.validate(input_val) def update_dtype(self, input_val): - if not self._is_composite: - return + if self._is_composite: + # Composite Datatypes & Arrays of Composite Datatypes - # Composite Datatypes & Arrays of Composite Datatypes + dt = self.dtype_tree.make_concrete_numpy_dtype(input_val) + + # Top level elements are not named and shape is + # already covered by the shape var + inner_dt = dt[1] - dt = self.dtype_tree.make_concrete_numpy_dtype(input_val) + if isinstance(self.raw_dtype, ArrayOf): + self.shape = dt[2] - # Top level elements are not named and shape is - # already covered by the shape var - dt = dt[1] + self.numpy_dtype = np.dtype(inner_dt) - self.numpy_dtype = np.dtype(dt) + self.dtype_str = self.numpy_dtype.str + self.dtype_descr = self.numpy_dtype.descr - self.dtype_str = self.numpy_dtype.str - self.describe_dict["dtype_str"] = self.dtype_str + return - self.dtype_descr = self.numpy_dtype.descr - self.describe_dict["dtype_descr"] = self.dtype_descr + if isinstance(self.raw_dtype, ArrayOf): + dt = self.dtype_tree.make_concrete_numpy_dtype(input_val) - self.describe_dict["dtype"] = self.dtype - self.describe_dict["shape"] = self.shape - self.describe_dict["SECOP_datainfo"] = self.secop_dtype_str + self.shape = dt[2] class SECoPReading: diff --git a/tests/test_dtype.py b/tests/test_dtype.py index 29b0b1a..062856f 100644 --- a/tests/test_dtype.py +++ b/tests/test_dtype.py @@ -1,84 +1,180 @@ -# import pytest -# from frappy.datatypes import ( -# ArrayOf, -# BLOBType, -# BoolType, -# EnumType, -# FloatRange, -# IntRange, -# ScaledInteger, -# StringType, -# StructOf, -# TupleOf, -# ) - -# from secop_ophyd.util import NestedRaggedArray, SECoPdtype - - -# def test_nested_ragged_inside_struct_arrays(): -# ragged = ArrayOf( -# StructOf(arr2=ArrayOf(FloatRange(), minlen=0, maxlen=100), name=StringType()), -# minlen=0, -# maxlen=100, -# ) - -# with pytest.raises(NestedRaggedArray): -# SECoPdtype(ragged) - - -# def test_nested_ragged_inside_tuple_arrays(): -# ragged = ArrayOf( -# TupleOf(ArrayOf(FloatRange(), minlen=0, maxlen=100), StringType()), -# minlen=0, -# maxlen=100, -# ) - -# with pytest.raises(NestedRaggedArray): -# SECoPdtype(ragged) - - -# def test_nested_ragged_arrays(): -# ragged = ArrayOf(ArrayOf(FloatRange(), -# minlen=0, maxlen=100), minlen=0, maxlen=100) - -# with pytest.raises(NestedRaggedArray): -# SECoPdtype(ragged) - - -# def test_nested_arrays(): - -# inner_dt_list = [ -# IntRange(), -# FloatRange(), -# ScaledInteger(1), -# StringType(), -# BoolType(), -# BLOBType(), -# EnumType("test", bla=0, blub=1), -# ] - -# dtype_list = ["number", "number", "number", -# "string", "boolean", "string", "number"] - -# for innner_dt, dtype in zip(inner_dt_list, dtype_list): - -# arr_dt = ArrayOf( -# ArrayOf(ArrayOf(innner_dt, minlen=5, maxlen=5), minlen=5, maxlen=5), -# minlen=0, -# maxlen=5, -# ) - -# sdtype = SECoPdtype(arr_dt) - -# assert sdtype.dtype == dtype - -# ragged_arr = ArrayOf( -# ArrayOf( -# ArrayOf(IntRange(min=0, max=100), minlen=0, maxlen=5), minlen=5, maxlen=5 -# ), -# minlen=5, -# maxlen=5, -# ) - -# with pytest.raises(NestedRaggedArray): -# SECoPdtype(ragged_arr) +import pytest +from frappy.client import CacheItem +from frappy.datatypes import ArrayOf, FloatRange, StringType, StructOf, TupleOf + +from secop_ophyd.util import SECoPdtype, SECoPReading + +RAGGED = True +REGULAR = False + + +@pytest.mark.parametrize( + "start_dtype,data,expected_shape,expected_update_shape", + [ + pytest.param( + ArrayOf(FloatRange(), minlen=0, maxlen=100), + [0.1, 0.1], + [100], + [2], + id="Ragged 1D", + ), + pytest.param( + ArrayOf(ArrayOf(FloatRange()), minlen=0, maxlen=100), + [[10, 10], [10, 10]], + [100, 100], # maxlen defaults to 100 if not given + [2, 2], + id="Ragged 2D", + ), + pytest.param( + ArrayOf(ArrayOf(ArrayOf(FloatRange())), minlen=0, maxlen=99), + [[[10, 10], [10, 10]], [[10, 10], [10, 10]]], + [99, 100, 100], + [2, 2, 2], + id="Ragged 3D", + ), + pytest.param( + ArrayOf( + ArrayOf(ArrayOf(FloatRange(), minlen=2, maxlen=2), minlen=2, maxlen=2), + minlen=2, + maxlen=2, + ), + [[[10, 10], [10, 10]], [[10, 10], [10, 10]]], + [2, 2, 2], + [2, 2, 2], + id="Regular 3D", + ), + pytest.param( + ArrayOf(TupleOf(ArrayOf(FloatRange(), minlen=0, maxlen=50), StringType())), + [([1, 2, 3], "blah")], + [100], + [1], + id="Complex Ragged 1D", + ), + pytest.param( + ArrayOf( + ArrayOf( + TupleOf(ArrayOf(FloatRange(), minlen=0, maxlen=50), StringType()) + ) + ), + [ + [([1, 2, 3], "blah"), ([1, 2, 3], "blah")], + [([1, 2, 3], "blah"), ([1, 2, 3], "blah")], + ], + [100, 100], + [2, 2], + id="Complex Ragged 2D", + ), + pytest.param( + TupleOf(ArrayOf(FloatRange(), minlen=0, maxlen=50), StringType()), + ([1, 2, 3], "blah"), + [], + [], + id="Complex", + ), + pytest.param( + StructOf( + mass=ArrayOf(FloatRange()), + pressure=ArrayOf(FloatRange()), + timestamp=ArrayOf(FloatRange()), + ), + {"mass": [], "pressure": [], "timestamp": []}, + [], + [], + id="struct of empty arrays", + ), + pytest.param( + ArrayOf(ArrayOf(FloatRange())), + [[], []], + [100, 100], + [2, 0], + id="empty 2d array ", + ), + pytest.param( + ArrayOf(ArrayOf(FloatRange())), + [[]], + [100, 100], + [1, 0], + id="empty 2d array [1,0]", + ), + pytest.param( + ArrayOf(ArrayOf(FloatRange())), + [], + [100, 100], + [0], + id="empty empty 2d", + ), + ], +) +def test_arrayof_update_dtype(start_dtype, data, expected_shape, expected_update_shape): + sdtype = SECoPdtype(start_dtype) + assert sdtype.shape == expected_shape + + sdtype.update_dtype(data) + assert sdtype.shape == expected_update_shape + + SECoPReading(sdtype, CacheItem(data, 234342.2, None, datatype=start_dtype)) + + +@pytest.mark.parametrize( + "start_dtype,expected_dtype_descr,expected_shape,max_depth", + [ + pytest.param( + TupleOf(ArrayOf(FloatRange(), maxlen=50), StringType()), + [("f0", "