From 5748170c23b2976781c02b080b68e19520df3fbf Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Thu, 13 Jan 2022 14:27:34 -0800 Subject: [PATCH 01/14] Implement summary variable --- examples/files/iris_epics_config.yml | 7 +++++++ lume_epics/epics_pva_server.py | 26 ++++++++++++++++++++++++++ lume_epics/utils.py | 18 ++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/examples/files/iris_epics_config.yml b/examples/files/iris_epics_config.yml index 185b7f7..dd4b08c 100644 --- a/examples/files/iris_epics_config.yml +++ b/examples/files/iris_epics_config.yml @@ -19,3 +19,10 @@ output_variables: Species: pvname: test:species protocol: pva + +summary: + pvname: test:iris:model + description: Iris example for lume-epics + owner: Jacqueline Garrahan + date_published: 1/13/2022 + id: model1 diff --git a/lume_epics/epics_pva_server.py b/lume_epics/epics_pva_server.py index f929a7c..1262a29 100644 --- a/lume_epics/epics_pva_server.py +++ b/lume_epics/epics_pva_server.py @@ -16,6 +16,7 @@ from p4p.server import Server as P4PServer from p4p.nt.ndarray import ntndarray as NTNDArrayData from p4p.server.raw import ServOpWrap +from p4p import Value, Type p4p_logger = logging.getLogger("p4p") p4p_logger.setLevel("DEBUG") @@ -285,6 +286,31 @@ def setup_server(self) -> None: else: self._providers[pvname] = None + if "summary" in self._epics_config: + pvname = self._epics_config["summary"].get("pvname") + owner = self._epics_config["summary"].get("owner") + date_published = self._epics_config["summary"].get("date_published") + description = self._epics_config["summary"].get("description") + id = self._epics_config["summary"].get("id") + + spec = [ + ("id", "s"), + ("owner", "s"), + ("date_published", "s"), + ("description", "s"), + ] + values = { + "id": id, + "date_published": date_published, + "description": description, + "owner": owner, + } + print(values) + pv_type = Type(id="summary", spec=spec) + value = Value(pv_type, values) + pv = SharedPV(initial=value) + self._providers[pvname] = pv + # initialize pva server self.pva_server = P4PServer(providers=[self._providers]) diff --git a/lume_epics/utils.py b/lume_epics/utils.py index 82bfd5f..9122393 100644 --- a/lume_epics/utils.py +++ b/lume_epics/utils.py @@ -56,4 +56,22 @@ def config_from_yaml(config_file): "protocol": protocol, } + if "summary" in config: + pvname = config["summary"].get("pvname") + owner = config["summary"].get("owner", "") + date_published = config["summary"].get("date_published", "") + description = config["summary"].get("description", "") + id = config["summary"].get("id", "") + + if not pvname: + raise ValueError("No pvname provided for summary variable.") + + epics_configuration["summary"] = { + "pvname": pvname, + "owner": owner, + "date_published": date_published, + "description": description, + "id": id, + } + return epics_configuration From da935063fd3a15fc6cba4ccb566ded50eb4cbd9e Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Fri, 14 Jan 2022 10:20:31 -0800 Subject: [PATCH 02/14] Add input and output served variables to pvAccess --- lume_epics/epics_pva_server.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lume_epics/epics_pva_server.py b/lume_epics/epics_pva_server.py index 1262a29..1d03bd6 100644 --- a/lume_epics/epics_pva_server.py +++ b/lume_epics/epics_pva_server.py @@ -298,14 +298,24 @@ def setup_server(self) -> None: ("owner", "s"), ("date_published", "s"), ("description", "s"), + ("input_variables", "as"), + ("output_variables", "as"), ] values = { "id": id, "date_published": date_published, "description": description, "owner": owner, + "input_variables": [ + self._epics_config[var]["pvname"] + for var in self._input_variables + ], + "output_variables": [ + self._epics_config[var]["pvname"] + for var in self._input_variables + ], } - print(values) + pv_type = Type(id="summary", spec=spec) value = Value(pv_type, values) pv = SharedPV(initial=value) From 8a15bfeb1bf96f2bf0a83af8889b772255e9bdbe Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Fri, 14 Jan 2022 10:21:06 -0800 Subject: [PATCH 03/14] Handle summary variable --- lume_epics/epics_server.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/lume_epics/epics_server.py b/lume_epics/epics_server.py index 7d85504..64c6c41 100644 --- a/lume_epics/epics_server.py +++ b/lume_epics/epics_server.py @@ -87,20 +87,15 @@ def __init__( self._protocols = [] ca_config = { - var: { - "pvname": self._epics_config[var]["pvname"], - "serve": self._epics_config[var]["serve"], - } + var: self._epics_config[var] for var in self._epics_config - if self._epics_config[var]["protocol"] in ["ca", "both"] + if self._epics_config[var].get("protocol") in ["ca", "both"] } pva_config = { - var: { - "pvname": self._epics_config[var]["pvname"], - "serve": self._epics_config[var]["serve"], - } + var: self._epics_config[var] for var in self._epics_config - if self._epics_config[var]["protocol"] in ["pva", "both"] + if self._epics_config[var].get("protocol") in ["pva", "both"] + or var == "summary" } if len(ca_config) > 0: From 81867e806b348a7d274a4cf2f38d612b5f471607 Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Wed, 26 Jan 2022 11:26:20 -0800 Subject: [PATCH 04/14] Check in fields for struct --- lume_epics/epics_pva_server.py | 149 ++++++++++++++++++++++++--------- lume_epics/epics_server.py | 45 ++++++++++ lume_epics/utils.py | 31 +++---- 3 files changed, 166 insertions(+), 59 deletions(-) diff --git a/lume_epics/epics_pva_server.py b/lume_epics/epics_pva_server.py index 1d03bd6..854f965 100644 --- a/lume_epics/epics_pva_server.py +++ b/lume_epics/epics_pva_server.py @@ -11,7 +11,7 @@ from functools import partial from lume_model.variables import InputVariable, OutputVariable from p4p.client.thread import Context -from p4p.nt import NTScalar, NTNDArray +from p4p.nt import NTScalar, NTNDArray, NTTable from p4p.server.thread import SharedPV from p4p.server import Server as P4PServer from p4p.nt.ndarray import ntndarray as NTNDArrayData @@ -189,6 +189,7 @@ def setup_server(self) -> None: try: val = self._context.get(self._varname_to_pvname_map[var_name]) val = val.raw.value + except: self.exit_event.set() raise ValueError( @@ -225,55 +226,123 @@ def setup_server(self) -> None: logger.info("Initializing pvAccess server") # initialize global inputs - for variable in variables.values(): - pvname = self._varname_to_pvname_map[variable.name] - - if self._epics_config[variable.name]["serve"]: - - # prepare scalar variable types - if variable.variable_type == "scalar": - nt = NTScalar("d") - initial = variable.value - - # prepare image variable types - elif variable.variable_type == "image": - nd_array = variable.value.view(NTNDArrayData) - nd_array.attrib = { - "x_min": variable.x_min, - "y_min": variable.y_min, - "x_max": variable.x_max, - "y_max": variable.y_max, - } - nt = NTNDArray() - initial = nd_array - - elif variable.variable_type == "array": - if variable.value_type == "str": - nt = NTScalar("as") + for variable_name, config in self._epics_config.items(): + + if config["serve"]: + + fields = config.get["fields"] + pvname = config.get["pvname"] + + if fields is not None: + + spec = [] + structure = {} + + for field in fields: + + variable = variables[field] + + if variable is None: + raise ValueError( + f"Field {field} for {variable_name} not found in variable list" + ) + + if variable.variable_type == "scalar": + spec.append((field, "d")) + nt = NTScalar("d") + initial = nt.wrap(variable.value) + + if variable.variable_type == "table": + spec.append((field, "v")) + nt = NTTable() + initial = nt.wrap(variable.value) + + if variable.variable_type == "array": + spec.append((field, "v")) + + if variable.value_type == "str": + nt = NTScalar("as") + initial = nt.wrap(variable.value) + + else: + nd_array = variable.value.view(NTNDArrayData) + nt = NTNDArray() + initial = nt.wrap(nd_array) + + if variable.variable_type == "image": + spec.append((field, "v")) + + nd_array = variable.value.view(NTNDArrayData) + nd_array.attrib = { + "x_min": variable.x_min, + "y_min": variable.y_min, + "x_max": variable.x_max, + "y_max": variable.y_max, + } + + nt = NTNDArray() + initial = nt.wrap(nd_array) + + structure[field] = initial + + # assemble pv + MyType = Type(id=variable_name, spec=spec) + struct_value = Value(MyType, structure) + pv = SharedPV(initial=struct_value) + self._providers[pvname] = pv + + else: + variable = variables[variable_name] + # prepare scalar variable types + if variable.variable_type == "scalar": + nt = NTScalar("d") initial = variable.value - else: + # prepare image variable types + elif variable.variable_type == "image": nd_array = variable.value.view(NTNDArrayData) + nd_array.attrib = { + "x_min": variable.x_min, + "y_min": variable.y_min, + "x_max": variable.x_max, + "y_max": variable.y_max, + } nt = NTNDArray() initial = nd_array - else: - raise ValueError( - "Unsupported variable type provided: %s", - variable.variable_type, - ) + elif variable.variable_type == "table": + nt = NTTable() + initial = nt.wrap(variable.value) - if variable.name in self._input_variables: - handler = PVAccessInputHandler( - pvname=pvname, is_constant=variable.is_constant, server=self - ) + elif variable.variable_type == "array": + if variable.value_type == "str": + nt = NTScalar("as") + initial = variable.value - pv = SharedPV(handler=handler, nt=nt, initial=initial) + else: + nd_array = variable.value.view(NTNDArrayData) + nt = NTNDArray() + initial = nd_array - else: - pv = SharedPV(nt=nt, initial=initial) + else: + raise ValueError( + "Unsupported variable type provided: %s", + variable.variable_type, + ) + + if variable.name in self._input_variables: + handler = PVAccessInputHandler( + pvname=pvname, + is_constant=variable.is_constant, + server=self, + ) + + pv = SharedPV(handler=handler, nt=nt, initial=initial) + + else: + pv = SharedPV(nt=nt, initial=initial) - self._providers[pvname] = pv + self._providers[pvname] = pv # if not serving pv, set up monitor else: diff --git a/lume_epics/epics_server.py b/lume_epics/epics_server.py index 64c6c41..9d4bcd7 100644 --- a/lume_epics/epics_server.py +++ b/lume_epics/epics_server.py @@ -84,6 +84,21 @@ def __init__( self._epics_config = epics_config + # define programatic access to model summary + self._pvname = None + self._owner = None + self._date_published = None + self._description = None + self._id = None + if "summary" in self._epics_config: + self._pvname = self._epics_config["summary"].get("pvname") + self._owner = self._epics_config["summary"].get("owner", "") + self._date_published = self._epics_config["summary"].get( + "date_published", "" + ) + self._description = self._epics_config["summary"].get("description", "") + self._id = self._epics_config["summary"].get("id", "") + self._protocols = [] ca_config = { @@ -309,3 +324,33 @@ def stop(self) -> None: self.pva_process.shutdown() logger.info("Server is stopped.") + + @property + def summary(self): + return { + "pvname": self._pvname, + "owner": self._owner, + "date published": self._date_published, + "description": self._description, + "id": self._id, + } + + @property + def owner(self): + return self._owner + + @property + def summary_pvname(self): + return self._pvname + + @property + def date_published(self): + return self._date_published + + @property + def description(self): + return self._description + + @property + def id(self): + return self._id diff --git a/lume_epics/utils.py b/lume_epics/utils.py index 9122393..fbf308d 100644 --- a/lume_epics/utils.py +++ b/lume_epics/utils.py @@ -19,36 +19,29 @@ def config_from_yaml(config_file): if "input_variables" in config: - for variable in config["input_variables"]: + # keep formatting distinction btw inputs/outputs for clarity + for variable in config["input_variables"] + config["output_variables"]: protocol = config["input_variables"][variable].get("protocol") serve = config["input_variables"][variable].get("serve", True) pvname = config["input_variables"][variable].get("pvname") + keys = list(config["output_variables"][variable].keys()) + if not protocol: raise ValueError(f"No protocol provided for {variable}") if not pvname: raise ValueError(f"No pvname provided for {variable}") - epics_configuration[variable] = { - "pvname": pvname, - "serve": serve, - "protocol": protocol, - } - - # Is this redundant? Do we need? - if "output_variables" in config: + keys.remove("protocol") + keys.remove("pvname") + try: + keys.remove("serve") + except ValueError: + pass - for variable in config["output_variables"]: - protocol = config["output_variables"][variable].get("protocol") - serve = config["output_variables"][variable].get("serve", True) - pvname = config["output_variables"][variable].get("pvname") - - if not protocol: - raise ValueError(f"No protocol provided for {variable}") - - if not pvname: - raise ValueError(f"No pvname provided for {variable}") + if len(keys) > 0 and protocol == "pva": + epics_configuration[variable]["fields"] = keys epics_configuration[variable] = { "pvname": pvname, From fe304139375d79789a965c5d024bf989206d3e64 Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Wed, 26 Jan 2022 14:26:38 -0800 Subject: [PATCH 05/14] Check in handling of table variables --- lume_epics/epics_pva_server.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lume_epics/epics_pva_server.py b/lume_epics/epics_pva_server.py index 854f965..00fb762 100644 --- a/lume_epics/epics_pva_server.py +++ b/lume_epics/epics_pva_server.py @@ -254,7 +254,13 @@ def setup_server(self) -> None: if variable.variable_type == "table": spec.append((field, "v")) - nt = NTTable() + table_rep = () + for col in variable.columns: + + # here we assume double type in tables... + table_rep += (col, "ad") + + nt = NTTable(table_rep) initial = nt.wrap(variable.value) if variable.variable_type == "array": @@ -311,7 +317,13 @@ def setup_server(self) -> None: initial = nd_array elif variable.variable_type == "table": - nt = NTTable() + table_rep = () + for col in variable.columns: + + # here we assume double type in tables... + table_rep += (col, "ad") + + nt = NTTable(table_rep) initial = nt.wrap(variable.value) elif variable.variable_type == "array": From 7532335f8164cb431c9c39cc06ac9b41ad85efa2 Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Wed, 26 Jan 2022 16:38:47 -0800 Subject: [PATCH 06/14] Check in test for PVA server construction --- lume_epics/tests/test_pva_server.py | 74 +++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 lume_epics/tests/test_pva_server.py diff --git a/lume_epics/tests/test_pva_server.py b/lume_epics/tests/test_pva_server.py new file mode 100644 index 0000000..efaa938 --- /dev/null +++ b/lume_epics/tests/test_pva_server.py @@ -0,0 +1,74 @@ +from lume_epics.epics_pva_server import PVAServer +from lume_model.variables import ( + ScalarInputVariable, + ImageInputVariable, + ArrayInputVariable, + ScalarOutputVariable, + ImageOutputVariable, + ArrayOutputVariable, + TableVariable, +) +import multiprocessing +import numpy as np + + +def test_pva_server(epics_config): + + table_data = { + "col1": ArrayInputVariable( + name="test", default=np.array([1, 2]), value_range=[0, 10] + ), + "col2": { + "row1": ScalarInputVariable( + name="col2_row1", default=0, value_range=[-1, -1] + ), + "row2": ScalarInputVariable( + name="col2_row2", default=0, value_range=[-1, 1] + ), + }, + } + + input_variables = { + "input1": ScalarInputVariable(name="input1", default=1.0, range=[0.0, 5.0]), + "input2": ScalarInputVariable( + name="input2", default=2.0, range=[0.0, 5.0], is_constant=True + ), + "input3": ImageInputVariable( + name="input3", + default=np.array([[1, 6,], [4, 1]]), + value_range=[1, 10], + axis_labels=["count_1", "count_2"], + x_min=0, + y_min=0, + x_max=5, + y_max=5, + ), + "input4": ArrayInputVariable( + name="input4", default=np.array([1, 2]), range=[0, 5] + ), + } + + output_variables = { + "output1": ScalarOutputVariable(name="output1"), + "output2": TableVariable(table_rows=["row1", "row2"], table_data=table_data), + "output3": ImageOutputVariable( + name="output3", axis_labels=["count_1", "count_2"], + ), + "output4": ArrayOutputVariable(name="output4"), + } + + in_queue = multiprocessing.Queue() + out_queue = multiprocessing.Queue() + running_indicator = multiprocessing.Value("b", False) + + server = PVAServer( + input_variables, + output_variables, + epics_config, + in_queue, + out_queue, + running_indicator, + ) + + server.start() + server.shutdown() From 92255d2ed5520b17de97d47021bbedc34ab3cb18 Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Wed, 26 Jan 2022 16:39:20 -0800 Subject: [PATCH 07/14] Rename struct --- lume_epics/epics_pva_server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lume_epics/epics_pva_server.py b/lume_epics/epics_pva_server.py index 00fb762..a30673b 100644 --- a/lume_epics/epics_pva_server.py +++ b/lume_epics/epics_pva_server.py @@ -292,8 +292,9 @@ def setup_server(self) -> None: structure[field] = initial # assemble pv - MyType = Type(id=variable_name, spec=spec) - struct_value = Value(MyType, structure) + struct_type = Type(id=variable_name, spec=spec) + + struct_value = Value(struct_type, structure) pv = SharedPV(initial=struct_value) self._providers[pvname] = pv From 13668108d8edba788edb9b9b87e2df081247ab7e Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Wed, 26 Jan 2022 16:40:05 -0800 Subject: [PATCH 08/14] Correct yaml parsing --- lume_epics/utils.py | 51 +++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/lume_epics/utils.py b/lume_epics/utils.py index fbf308d..0448104 100644 --- a/lume_epics/utils.py +++ b/lume_epics/utils.py @@ -6,7 +6,7 @@ def config_from_yaml(config_file): - """ + """Load yaml file into configuration """ config = yaml.safe_load(config_file) @@ -17,37 +17,38 @@ def config_from_yaml(config_file): epics_configuration = {} - if "input_variables" in config: + variables = config["input_variables"] + variables.update(config["output_variables"]) - # keep formatting distinction btw inputs/outputs for clarity - for variable in config["input_variables"] + config["output_variables"]: - protocol = config["input_variables"][variable].get("protocol") - serve = config["input_variables"][variable].get("serve", True) - pvname = config["input_variables"][variable].get("pvname") + # keep formatting distinction btw inputs/outputs for clarity + for variable, var_config in variables.items(): + protocol = var_config.get("protocol") + serve = var_config.get("serve", True) + pvname = var_config.get("pvname") - keys = list(config["output_variables"][variable].keys()) + keys = list(var_config.keys()) - if not protocol: - raise ValueError(f"No protocol provided for {variable}") + if not protocol: + raise ValueError(f"No protocol provided for {variable}") - if not pvname: - raise ValueError(f"No pvname provided for {variable}") + if not pvname: + raise ValueError(f"No pvname provided for {variable}") - keys.remove("protocol") - keys.remove("pvname") - try: - keys.remove("serve") - except ValueError: - pass + keys.remove("protocol") + keys.remove("pvname") + try: + keys.remove("serve") + except ValueError: + pass - if len(keys) > 0 and protocol == "pva": - epics_configuration[variable]["fields"] = keys + if len(keys) > 0 and protocol == "pva": + epics_configuration[variable]["fields"] = keys - epics_configuration[variable] = { - "pvname": pvname, - "serve": serve, - "protocol": protocol, - } + epics_configuration[variable] = { + "pvname": pvname, + "serve": serve, + "protocol": protocol, + } if "summary" in config: pvname = config["summary"].get("pvname") From 97b0f2bab1761447e838a3ad46c55ff0faf62556 Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Thu, 27 Jan 2022 11:08:16 -0800 Subject: [PATCH 09/14] Fix field parsing and serving --- lume_epics/epics_pva_server.py | 6 +++--- lume_epics/epics_server.py | 9 ++++++++- lume_epics/utils.py | 15 ++++----------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lume_epics/epics_pva_server.py b/lume_epics/epics_pva_server.py index a30673b..502b0f7 100644 --- a/lume_epics/epics_pva_server.py +++ b/lume_epics/epics_pva_server.py @@ -230,8 +230,8 @@ def setup_server(self) -> None: if config["serve"]: - fields = config.get["fields"] - pvname = config.get["pvname"] + fields = config.get("fields") + pvname = config.get("pvname") if fields is not None: @@ -250,7 +250,7 @@ def setup_server(self) -> None: if variable.variable_type == "scalar": spec.append((field, "d")) nt = NTScalar("d") - initial = nt.wrap(variable.value) + initial = variable.value if variable.variable_type == "table": spec.append((field, "v")) diff --git a/lume_epics/epics_server.py b/lume_epics/epics_server.py index 9d4bcd7..a0a1617 100644 --- a/lume_epics/epics_server.py +++ b/lume_epics/epics_server.py @@ -113,6 +113,12 @@ def __init__( or var == "summary" } + # track nested fields + self._pva_fields = [] + for var, config in self._epics_config.items(): + if config.get("fields"): + self._pva_fields += config["fields"] + if len(ca_config) > 0: self._protocols.append("ca") @@ -264,7 +270,8 @@ def run_comm_thread( outputs = [ var for var in predicted_output - if self._epics_config[var.name]["protocol"] + if var.name in self._pva_fields + or self._epics_config[var.name]["protocol"] in [protocol, "both"] ] queue.put({"output_variables": outputs}, timeout=0.1) diff --git a/lume_epics/utils.py b/lume_epics/utils.py index 0448104..0606037 100644 --- a/lume_epics/utils.py +++ b/lume_epics/utils.py @@ -26,23 +26,13 @@ def config_from_yaml(config_file): serve = var_config.get("serve", True) pvname = var_config.get("pvname") - keys = list(var_config.keys()) - if not protocol: raise ValueError(f"No protocol provided for {variable}") if not pvname: raise ValueError(f"No pvname provided for {variable}") - keys.remove("protocol") - keys.remove("pvname") - try: - keys.remove("serve") - except ValueError: - pass - - if len(keys) > 0 and protocol == "pva": - epics_configuration[variable]["fields"] = keys + fields = var_config.get("fields") epics_configuration[variable] = { "pvname": pvname, @@ -50,6 +40,9 @@ def config_from_yaml(config_file): "protocol": protocol, } + if fields: + epics_configuration[variable]["fields"] = fields + if "summary" in config: pvname = config["summary"].get("pvname") owner = config["summary"].get("owner", "") From e6167bb763af57ccfb0f6d5dbf9f91e59d403d39 Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Thu, 27 Jan 2022 11:08:33 -0800 Subject: [PATCH 10/14] Add configuration file --- lume_epics/tests/files/epics_config_struct.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 lume_epics/tests/files/epics_config_struct.yml diff --git a/lume_epics/tests/files/epics_config_struct.yml b/lume_epics/tests/files/epics_config_struct.yml new file mode 100644 index 0000000..7f91559 --- /dev/null +++ b/lume_epics/tests/files/epics_config_struct.yml @@ -0,0 +1,18 @@ +input_variables: + input1: + pvname: test:input1 + protocol: pva + + input2: + pvname: test:input2 + protocol: pva + + +output_variables: + output_summary: + pvname: test:output_summary + protocol: pva + fields: + - output2 + - output3 + - output4 From 6763d44fce4e470297f5a0b878145f6fdda574b5 Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Thu, 27 Jan 2022 11:08:56 -0800 Subject: [PATCH 11/14] Check in new tests --- lume_epics/tests/conftest.py | 9 ++++- lume_epics/tests/launch_server.py | 17 +++++++- lume_epics/tests/test_pva_server.py | 62 +++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/lume_epics/tests/conftest.py b/lume_epics/tests/conftest.py index 2ea204d..44427a7 100644 --- a/lume_epics/tests/conftest.py +++ b/lume_epics/tests/conftest.py @@ -61,13 +61,20 @@ def tear_down(): @pytest.fixture(scope="session", autouse=True) def epics_config(rootdir): - # with open(f"{rootdir}/lume_epics/tests/files/epics_config.yml", "r") as f: with open(f"{rootdir}/files/epics_config.yml", "r") as f: epics_config = config_from_yaml(f) yield epics_config +@pytest.fixture(scope="session", autouse=True) +def epics_config_struct(rootdir): + with open(f"{rootdir}/files/epics_config_struct.yml", "r") as f: + epics_config = config_from_yaml(f) + + yield epics_config + + @pytest.fixture(scope="session", autouse=True) def server(): # diff --git a/lume_epics/tests/launch_server.py b/lume_epics/tests/launch_server.py index d41ea7e..8ef6378 100644 --- a/lume_epics/tests/launch_server.py +++ b/lume_epics/tests/launch_server.py @@ -4,7 +4,14 @@ import sys from lume_epics import epics_server from lume_model.models import SurrogateModel -from lume_model.variables import * +from lume_model.variables import ( + ScalarInputVariable, + ImageInputVariable, + ScalarOutputVariable, + ArrayInputVariable, + ImageOutputVariable, + ArrayOutputVariable, +) from lume_epics.utils import config_from_yaml logger = logging.getLogger(__name__) @@ -54,27 +61,35 @@ def evaluate(self, input_variables): self.output_variables["output3"].value = ( self.input_variables["input3"].value * 2 ) + self.output_variables["output3"].x_min = ( self.input_variables["input3"].x_min / 2 ) + self.output_variables["output3"].x_max = ( self.input_variables["input3"].x_max / 2 ) + self.output_variables["output3"].y_min = ( self.input_variables["input3"].y_min / 2 ) + self.output_variables["output3"].y_max = ( self.input_variables["input3"].y_max / 2 ) + self.output_variables["output3"].x_min = ( self.input_variables["input3"].x_min / 2 ) + self.output_variables["output3"].x_max = ( self.input_variables["input3"].x_max / 2 ) + self.output_variables["output3"].y_min = ( self.input_variables["input3"].y_min / 2 ) + self.output_variables["output3"].y_max = ( self.input_variables["input3"].y_max / 2 ) diff --git a/lume_epics/tests/test_pva_server.py b/lume_epics/tests/test_pva_server.py index efaa938..593d3c1 100644 --- a/lume_epics/tests/test_pva_server.py +++ b/lume_epics/tests/test_pva_server.py @@ -72,3 +72,65 @@ def test_pva_server(epics_config): server.start() server.shutdown() + + +def test_pva_server_struct(epics_config_struct): + + table_data = { + "col1": ArrayInputVariable( + name="test", default=np.array([1, 2]), value_range=[0, 10] + ), + "col2": { + "row1": ScalarInputVariable( + name="col2_row1", default=0, value_range=[-1, -1] + ), + "row2": ScalarInputVariable( + name="col2_row2", default=0, value_range=[-1, 1] + ), + }, + } + + input_variables = { + "input1": ScalarInputVariable(name="input1", default=1.0, range=[0.0, 5.0]), + "input2": ScalarInputVariable( + name="input2", default=2.0, range=[0.0, 5.0], is_constant=True + ), + "input3": ImageInputVariable( + name="input3", + default=np.array([[1, 6,], [4, 1]]), + value_range=[1, 10], + axis_labels=["count_1", "count_2"], + x_min=0, + y_min=0, + x_max=5, + y_max=5, + ), + "input4": ArrayInputVariable( + name="input4", default=np.array([1, 2]), range=[0, 5] + ), + } + + output_variables = { + "output1": ScalarOutputVariable(name="output1"), + "output2": TableVariable(table_rows=["row1", "row2"], table_data=table_data), + "output3": ImageOutputVariable( + name="output3", axis_labels=["count_1", "count_2"], + ), + "output4": ArrayOutputVariable(name="output4"), + } + + in_queue = multiprocessing.Queue() + out_queue = multiprocessing.Queue() + running_indicator = multiprocessing.Value("b", False) + + server = PVAServer( + input_variables, + output_variables, + epics_config_struct, + in_queue, + out_queue, + running_indicator, + ) + + server.start() + server.shutdown() From 703476ebc29ab3eff346d39542e0fdfd250568a9 Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Thu, 27 Jan 2022 11:11:59 -0800 Subject: [PATCH 12/14] Check in examples --- examples/files/epics_config_struct.yml | 17 +++++++++++++++++ examples/keras/iris_server.py | 21 ++++++++++----------- examples/struct/server.py | 26 ++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 examples/files/epics_config_struct.yml create mode 100644 examples/struct/server.py diff --git a/examples/files/epics_config_struct.yml b/examples/files/epics_config_struct.yml new file mode 100644 index 0000000..37a0513 --- /dev/null +++ b/examples/files/epics_config_struct.yml @@ -0,0 +1,17 @@ +input_variables: + input1: + pvname: test:input1 + protocol: ca + + input2: + pvname: test:input2 + protocol: pva + +output_variables: + output_summary: + pvname: test:output_summary + protocol: pva + fields: + - output1 + - output2 + - output3 diff --git a/examples/keras/iris_server.py b/examples/keras/iris_server.py index 70de64d..291f72d 100644 --- a/examples/keras/iris_server.py +++ b/examples/keras/iris_server.py @@ -1,16 +1,15 @@ - from lume_epics.epics_server import Server from lume_model.utils import model_from_yaml +from lume_epics.utils import config_from_yaml + +if __name__ == "__main__": + with open("examples/files/iris_config.yml", "r") as f: + model_class, model_kwargs = model_from_yaml(f, load_model=False) -with open("examples/files/iris_config.yml", "r") as f: - model_class, model_kwargs = model_from_yaml(f, load_model=False) + with open("examples/files/iris_epics_config.yml", "r") as f: + epics_config = config_from_yaml(f) -prefix = "test" -server = Server( - model_class, - prefix, - model_kwargs=model_kwargs -) + server = Server(model_class, epics_config, model_kwargs=model_kwargs) -# monitor = False does not loop in main thread -server.start(monitor=True) \ No newline at end of file + # monitor = False does not loop in main thread + server.start(monitor=True) diff --git a/examples/struct/server.py b/examples/struct/server.py new file mode 100644 index 0000000..3465118 --- /dev/null +++ b/examples/struct/server.py @@ -0,0 +1,26 @@ +from lume_epics.epics_server import Server +from lume_model.utils import variables_from_yaml +from lume_epics.utils import config_from_yaml +import os +from examples.model import DemoModel + +import logging + + +if __name__ == "__main__": + with open("examples/files/demo_config.yml", "r") as f: + input_variables, output_variables = variables_from_yaml(f) + + with open("examples/files/epics_config_struct.yml", "r") as f: + epics_config_struct = config_from_yaml(f) + + server = Server( + DemoModel, + epics_config_struct, + model_kwargs={ + "input_variables": input_variables, + "output_variables": output_variables, + }, + ) + # monitor = False does not loop in main thread + server.start(monitor=True) From 3a427df442b44de2dd30445d9c6c73ab5f76737b Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Thu, 27 Jan 2022 11:13:33 -0800 Subject: [PATCH 13/14] Raise value error when using fields with channel access --- lume_epics/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lume_epics/utils.py b/lume_epics/utils.py index 0606037..c576951 100644 --- a/lume_epics/utils.py +++ b/lume_epics/utils.py @@ -34,6 +34,9 @@ def config_from_yaml(config_file): fields = var_config.get("fields") + if fields is not None and protocol == "ca": + raise ValueError("Cannot serve fields with Channel Access server.") + epics_configuration[variable] = { "pvname": pvname, "serve": serve, From e8d13b8cb96fdc516444fb2eeedc54509263c25e Mon Sep 17 00:00:00 2001 From: Jacqueline Garrahan Date: Thu, 27 Jan 2022 11:39:32 -0800 Subject: [PATCH 14/14] Update configuration with summary/structure --- docs/EPICS.md | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/docs/EPICS.md b/docs/EPICS.md index 6323ae1..32c2c45 100644 --- a/docs/EPICS.md +++ b/docs/EPICS.md @@ -33,6 +33,70 @@ The optional field `serve` for each variable accepts a boolean defaulting to tru The client controller `lume_epics.client.controller.Controller` is also initialized using the EPICS configuration dictionary and a common file may be used for a project, though the serve field is unimportant to the controller. +Over pvAccess, you also have the option to host a summary process variable: + +```yaml +input_variables: + input1: + pvname: test:input1 + protocol: ca + serve: false + + input2: + pvname: test:input2 + protocol: pva + +output_variables: + output1: + pvname: test:output1 + protocol: pva + + output2: + pvname: test:output2 + protocol: pva + + output3: + pvname: test:output3 + protocol: pva + +summary: + pvname: test:summary_variable + owner: Jacqueline Garrahan + date_published: 1/27/22 + description: A basic epics configuration + id: model1 +``` + + +You can also serve output variables as a pvAccess structure: + +```yaml +input_variables: + input1: + pvname: test:input1 + protocol: ca + serve: false + + input2: + pvname: test:input2 + protocol: pva + +output_variables: + pvname: test:output + protocol: pva + fields: + - output1 + - output2 + - output3 + +summary: + pvname: test:summary_variable + owner: Jacqueline Garrahan + date_published: 1/27/22 + description: A basic epics configuration + id: model1 +``` + ## EPICS environment configuration