Merge branch 'mxcube:develop' into p11_develop

.github/workflows/coverage_report.yml

@@ -22,7 +22,7 @@ jobs:
         sudo apt-get update && sudo apt-get install -y libsasl2-dev libldap2-dev libssl-dev
         python -m pip install --upgrade pip
         python -m pip install poetry --user
-        python -m poetry install
+        python -m poetry install --extras=tango
 
     - name: Run and write pytest
       run: |

.github/workflows/pages.yaml

@@ -5,7 +5,6 @@ name: "Pages"
 
 concurrency:
   group: "pages"
-  cancel-in-progress: true
 
 on:
   push:

.github/workflows/tests.yml

@@ -25,7 +25,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install poetry --user
-          python -m poetry install
+          python -m poetry install --extras=tango
 
       - name: Test with pytest
         run: |

.pre-commit-config.yaml

@@ -5,9 +5,7 @@ repos:
     rev: 1.5.0
     hooks:
       - id: poetry-check
-        language_version: python3.8
 #      - id: poetry-lock
-#        language_version: python3.8
   - repo: https://github.com/myint/autoflake
     rev: v1.6.0
     hooks:
@@ -24,7 +22,6 @@ repos:
     hooks:
     - id: black
       name: Black
-      language_version: python3.8
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.3.0
     hooks:

.readthedocs.yaml

@@ -20,9 +20,9 @@ build:
     python: "mambaforge-22.9"
 
   commands:
-    - "mamba env create --file conda-environment-dev.yml"
+    - "mamba env create --file conda-environment-dev.yml --force"
     - "mamba run --name mxcubecore poetry install --only=docs,main"
-    - "mamba run --name mxcubecore poetry run sphinx-build -T -E -b html -c ./docs/ ./docs/source/ ${READTHEDOCS_OUTPUT}/html/"
+    - "mamba run --name mxcubecore sphinx-build -T -E -b html -c ./docs/ ./docs/source/ ${READTHEDOCS_OUTPUT}/html/"
 
 
 ...  # EOF

README.md

@@ -2,11 +2,20 @@
 [![Test and build](https://github.com/mxcube/mxcubecore/actions/workflows/tests.yml/badge.svg)](https://github.com/mxcube/mxcubecore/actions/workflows/tests.yml)
 ![PyPI](https://img.shields.io/pypi/v/mxcubecore)
 
+
 # Backend of MXCuBE
-mxcubecore is the back-end level of [MXCuBE Qt](https://github.com/mxcube/mxcube/) and [MXCuBE Web](https://github.com/mxcube/mxcube3/)  allowing to access the beamline control system and instrumentation. It acts as a container of single python objects (called hardware objects).
 
-Please read the [contributing guidlines](https://github.com/mxcube/mxcubecore/blob/master/CONTRIBUTING.md/) before submiting your code to the repository.
+`mxcubecore` is the back-end library for
+[MXCuBE Qt](https://github.com/mxcube/mxcubeqt/)
+and [MXCuBE Web](https://github.com/mxcube/mxcubeweb/).
+It allows access to the beamline control system and instrumentation.
+It acts as a container of single Python objects (called "hardware objects").
+
+Please read the
+[contributing guidelines](https://mxcubecore.readthedocs.io/en/stable/dev/contributing.html)
+before submitting your code to the repository.
 
+## License
 
 MXCuBE is free software: you can redistribute it and/or modify
 it under the terms of the GNU Lesser General Public License as published by
@@ -21,10 +30,23 @@ GNU Lesser General Public License for more details.
 You should have received a copy of the GNU Lesser General Public License
 along with MXCuBE. If not, see <https://www.gnu.org/licenses/>.
 
-# Installation
 
-Installation of the mxubecore module is commonly done as a depdency of either [MXCuBE Web](https://github.com/mxcube/mxcube3/) or [MXCuBE Qt](https://github.com/mxcube/mxcube/).
+## Installation
+
+Installation of the `mxcubecore` module is commonly done as a dependency of either
+[MXCuBE Web](https://github.com/mxcube/mxcubeweb/)
+or [MXCuBE Qt](https://github.com/mxcube/mxcubeqt/).
+
+Standalone installation of the `mxcubecore` for development purposes can be done
+via Poetry with `poetry install`
+or via pip with `python -m pip install --editable .` (don't forget the trailing dot `.`).
 
-Standalone installation of the moudle can be done via poetry `poetry install` or for development purpouses (creating a symlink to the current folder) by using pip: `pip install -e .`. Don't forget the trailing "." (period).
+`mxcubecore` depends on `python-ldap` that in turn depends on the `openldap` system library.
+It can be installed in a conda environment: `conda install openldap`
+or on systems using the apt package manager (Debian and derivatives, including Ubuntu):
+`sudo apt-get install -y libldap2-dev libsasl2-dev`.
+See [python-ldap](https://www.python-ldap.org/en/python-ldap-3.4.3/installing.html#debian)
+for more information.
 
-mxcubecore depends on python-ldap that in turn depends on the openldap system library. It can be installed through conda by installing the openldap depdency: `conda install openldap` or on systems using apt-get: `sudo apt-get install -y libldap2-dev libsasl2-dev slapd ldap-utils tox lcov valgrind`. See [python-ldap](https://www.python-ldap.org/en/python-ldap-3.4.3/installing.html#debian) for more information.
+See [Developer documentation](https://mxcubecore.readthedocs.io/)
+for more information on working with mxcubecore.

conda-environment-dev.yml

@@ -2,18 +2,21 @@ name: mxcubecore
 channels:
   - conda-forge
 dependencies:
+
+  ### Runtime dependencies
+
   - python >=3.8,<3.11
-  # Run-time
-  - openldap
-  - python-ldap=3.4.0
-  # Dev
-  - pip
+
+  # We install `python-ldap` from conda because it is "hard" to install.
+  # On PyPI it is not available as *wheel*, only as *sdist* which requires to be built
+  # with compilation steps that require a compiler and some header files.
+  # Installing with conda is much "easier".
+  #
+  # IMPORTANT:
+  # Always make sure that `python-ldap` here is pinned to the same version
+  # as in the Poetry lockfile `poetry.lock` (not the other way around).
+  - python-ldap=3.4.3
+
+  ### Development dependencies
+
   - poetry
-  # Format, lint
-  - black
-  - flake8
-  # Test
-  - coverage
-  - pytest
-  - pytest-cov
-  - pytest-mock

docs/conf.py

@@ -29,9 +29,7 @@
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
-extensions = [
-    "myst_parser",
-]
+extensions = []
 
 root_doc = "contents"
 
@@ -98,4 +96,11 @@
 extensions.append("sphinx.ext.viewcode")
 
 
+# -- Options for myst_parser
+
+extensions.append("myst_parser")
+
+myst_enable_extensions = ("fieldlist",)
+
+
 # EOF

docs/source/dev/index.rst

@@ -9,6 +9,7 @@ Developer documentation
 .. toctree::
     :caption: Contents:
     :glob:
+    :maxdepth: 2
     :titlesonly:
 
     *

docs/source/dev/json-schema-generated-user-interface.md

@@ -0,0 +1,97 @@
+# JSON-schema generated user interface
+
+:Created: Rasmus Fogh, 20230514
+:Updated:
+  - 20230730
+  - 20230927
+
+After the PRs for [mxcubecore #755](https://github.com/mxcube/mxcubecore/pull/755) and [mxcubeqt #442](https://github.com/mxcube/mxcubeqt/pull/442) we have a system for code-generated interfaces that can be plugged in to both a Qt and a Web user interface. It is implemented for Qt5, and with some help it should be possible to implement a (prototype?) web interface on the same basis. After discussion with [@marcusoscarsson](https://github.com/marcus-oscarsson) the PR is now refactored to do parameter updating on the server side only, sending values back and forth as necessary through signals. The newest version is through PRs [#794 (mxcubecore)](https://github.com/mxcube/mxcubecore/pull/794) and [#453 (mxcubeqt)](https://github.com/mxcube/mxcubeqt/pull/453).
+
+The system is used to generate parameter queries from mxcubecore (for now from the GΦL workflow). It includes a wide range of widgets; multiple nested layout fields; support for custom update functions triggered when values are edited, that can reset values of other fields or change widget colouring; field value validation; and colouring and disabling of action for invalid widgets. There are several examples of pulldown menu contents being modified depending on the values of other fields. The system is started with a `PARAMETERS_NEEDED` on signals being sent from the mxcubecore side with the JSON schemas as parameters. The UI side responds by sending a `PARAMETER_RETURN_SIGNAL` with the complete dictionary of parameter values. There is also a control parameter, that determines whether the response matches a Continue or Cancel click (closing the UI popup), or whether it is a request for a UI update. In the latter case the update is sent from the mxcubecore side with a `PARAMETER_UPDATE_SIGNAL`.
+
+I have done my best to follow the correct practice of using JSON schemas to specify user interfaces – but possibly with mixed success. The documentation I could find was neither as clear nor as comprehensive as is the case for Qt (where it is already hard to find what you need), and the capabilities of the proposed system go rather beyond what you see in web examples. The main documentation used has been <https://rjsf-team.github.io/react-jsonschema-form/>, <https://github.com/jsonform/jsonform/wiki>, <https://www.npmjs.com/package/react-jsonschema-form-layout>, <https://ui-schema.bemit.codes/>, <https://react-jsonschema-form.readthedocs.io/en/v1.8.1/form-customization/>. Ultimately I had to invent a certain amount, which may then be inconsistent with web libraries and best practices. Hopefully someone more familiar with JSON-schema and web practice would be able to find a more correct way of handling some of the problems. One advantage is that since this system is purely internal to MXCuBE-Qt so far it can (and will) be changed to conform to the needs of the web implementation.
+
+The only really detailed documentation available is the Qt implementation, which serves as a worked example. The attached files show screenshots of the generated interfaces (with some examples of e.g. widget colouring); the JSON schemas that generate them can be found in the mxcubecore repository, in `mxcubecore/doc/json_schema_gui` as can the parameters returned, and some examples of the response to gui update requests.
+
+
+## Implementation
+
+The current implementation is found in the following files. The files in mxcubecore serve for both Qt and web interfaces, whereas the files on the Qt side would need to be reimplemented/replaced for a Web interface.
+
+
+### `mxcubecore/HardwareObjects/GphlWorkflow.py`
+
+This is the client, essentially. The calls to the user interface happen entirely within then two functions `query_pre_strategy_params` and `query_collection_strategy`. `GphlWorkflow.py` also contains a number of methods `update_xyz` and `adjust_xyz` that take care of UI update requests. Signals from the UI side are received and dispatched by the `receive_pre_strategy_data` and `receive_pre_collection_data` methods, connected to signals.
+
+
+### `mxcubeqt/widgets/gphl_json_dialog.py`
+
+This contains the function that is triggered by receiving the signal (`gphlJsonParametersNeeded`), sets up the UI window with Continue button etc., and calls the function that creates the contained UI widgets.
+
+
+### `mxcubeqt/widgets/jsonparamsgui.py`
+
+This contains all the widgets, and the UI creation function (create_widgets) that transforms the input schemas to a UI. The top level widget is the LayoutWidget class. Widgets include Layout widgets (VerticalBox, HorizontalBox, and ColumnGridWqidget for gridded layouts), standard widgets for primitive types, and SelectionTable.
+
+
+## Attached Examples
+
+The JSON schemas that give rise so the images shown here can be found in the mxcubecore repository, github develop branch, in `mxcubecore/doc/json_schema_gui`.
+
+
+### PreCharacterisation
+
+Called from `query_pre_strategy_params`
+
+![PreCharacterisation](/assets/pre-characterisation.jpeg)
+
+
+### Characterisation
+
+Called from `query_collection_strategy`
+
+![Characterisation](/assets/characterisation.jpeg)
+
+
+### PreAcquisition
+
+Called from `query_pre_strategy_params`
+
+Note that row 4 is selected automatically (row number shown in bold). Manually selected rows are also shown with a blue border. This could be improved.
+
+![PreAcquisition](/assets/pre-acquisition.jpeg)
+
+
+### Acquisition
+
+Called from `query_collection_strategy`
+
+![Acquisition](/assets/acquisition.jpeg)
+
+
+### PreAcquisition_2
+
+Called from `query_pre_strategy_params`
+
+Another example of a pre-acquisition popup. Note that in this image row 12 is selected. The update information generated (successively) by selecting this row, by setting Crystal Lattice to 'Cubic', and by setting Space Group to F432 can be found in `update_indexing.json`, `update_lattice.json`, and `update_spacegroup.json`, respectively (in `mxcubecore/doc/json_schema_gui`)
+
+![PreAcquisition_2](/assets/pre-acquisition-2.jpeg)
+
+
+### Acquisition_3
+
+Called from `query_collection_strategy`
+
+Note that this is another run than `PreAcquisition_2`. This being a MAD experiment, two additional wavelengths are shown in the UI.
+The dose and dose budget fields are coloured yellow as a warning because after increasing the transmission value, the dose value is higher than the dose budget. Editing the transmission field has also coloured that filed orange, as 'most recently edited'. Note that the Continue button is still active. This is done through update functions. The update information generated by changing the transmission can be found in `update_transmission.json` (in `mxcubecore/doc/json_schema_gui`)
+
+![Acquisition_3](/assets/acquisition-3.jpeg)
+
+
+### PreDiffractometerCalibration
+
+Called from `query_pre_strategy_params`
+
+Note that cell parameters are editable in PreDiffractometerCalibration (unlike in the very similar PreCharacterisation popup) (and have been edited in the snapshot, most recent edit showing in orange). The resolution has been set to an illegal value, colouring the field light red, and disabling the Continue button.The resolution was reset to a valid value before producing the `result.json` file.
+
+![PreDiffractometerCalibration](/assets/pre-diffractometer-calibration.jpeg)

mxcubecore/Command/Sardana.py

@@ -429,9 +429,11 @@ def init_device(self):
                     self.info.maxval = float(ranges[-1])
             elif int(taurus.Release.version[0]) > 3:  # taurus 4 and beyond
                 minval, maxval = self.attribute.getRange()
-                if minval:
+                if minval is not None:
                     self.info.minval = minval.magnitude
+                if maxval is not None:
                     self.info.maxval = maxval.magnitude
+
         except Exception:
             import traceback
 

mxcubecore/HardwareObjects/Beamline.py

@@ -27,7 +27,8 @@
 from __future__ import division, absolute_import
 from __future__ import print_function, unicode_literals
 
-from typing import Union
+from typing import Union, Any
+from mxcubecore.dispatcher import dispatcher
 
 __copyright__ = """ Copyright © 2019 by the MXCuBE collaboration """
 __license__ = "LGPLv3+"
@@ -124,6 +125,16 @@ def __init__(self, name):
         # List of undulators
         self.undulators = []
 
+        # Format of mesh result for display
+        self.mesh_result_format = "PNG"
+
+        # bool Use the native mesh feature available, true by default
+        self.use_native_mesh = True
+
+        # bool Enable features to work with points in the plane, called
+        # 2D-points, (none centred positions)
+        self.enable_2d_points = True
+
         # Dictionary with the python id of hardwareobject as key
         # and the "dotted/attribute path" to hardwareobject from the
         # Beamline object
@@ -229,6 +240,36 @@ def _get_id_dict_rec(
 
         return _path
 
+
+    # Signal handling functions:
+    def emit(self, signal: Union[str, object, Any], *args) -> None:
+        """Emit signal. Accepts both multiple args and a single tuple of args.
+
+        This is needed for communication from the GUI to the core
+        (jsonparamsgui in mxcubeqt)
+
+        NBNB TODO HACK
+        This is a duplicate of the same function in HardwareObjectMixin.
+        Since the Beamline is not a CommandContainer or a normal HardwareObject
+        it may not be appropriate to make it a subclass of HardwareObjectYaml
+        We need to consider how we want this organised
+
+        Args:
+            signal (Union[str, object, Any]): In practice a string, or dispatcher.
+            *args (tuple): Arguments sent with signal.
+        """
+
+        signal = str(signal)
+
+        if len(args) == 1:
+            if isinstance(args[0], tuple):
+                args = args[0]
+        responses: list = dispatcher.send(signal, self, *args)
+        if not responses:
+            raise RuntimeError(
+                "Signal %s is not connected" % signal
+            )
+
     # NB this function must be re-implemented in nested subclasses
     @property
     def all_roles(self):
@@ -250,6 +291,18 @@ def machine_info(self):
 
     __content_roles.append("machine_info")
 
+
+    @property
+    def authenticator(self):
+        """Authenticator Hardware object
+
+        Returns:
+            Optional[AbstractAuthenticator]:
+        """
+        return self._objects.get("authenticator")
+
+    __content_roles.append("authenticator")
+
     @property
     def transmission(self):
         """Transmission Hardware object