From 2b51ae1b28b66ae3ce789480be94caf6112758bd Mon Sep 17 00:00:00 2001 From: tangkong Date: Thu, 11 Jul 2024 11:44:51 -0700 Subject: [PATCH 1/8] MNT: gather available backends into BACKENDS dictionary --- superscore/backends/__init__.py | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/superscore/backends/__init__.py b/superscore/backends/__init__.py index e69de29..4d5a579 100644 --- a/superscore/backends/__init__.py +++ b/superscore/backends/__init__.py @@ -0,0 +1,38 @@ +__all__ = ['BACKENDS'] + +import logging +from typing import Dict + +from .core import _Backend + +logger = logging.getLogger(__name__) + + +def _get_backend(backend: str) -> _Backend: + if backend == 'filestore': + from .filestore import FilestoreBackend + return FilestoreBackend + if backend == 'test': + from .test import TestBackend + return TestBackend + + raise ValueError(f"Unknown backend {backend}") + + +def _get_backends() -> Dict[str, _Backend]: + backends = {} + + try: + backends['filestore'] = _get_backend('filestore') + except ImportError as ex: + logger.debug(f"Filestore Backend unavailable: {ex}") + + try: + backends['test'] = _get_backend('test') + except ImportError as ex: + logger.debug(f"Test Backend unavailable: {ex}") + + return backends + + +BACKENDS = _get_backends() From 7d9a4ae5f184768505e9e3017b44d32a625b653b Mon Sep 17 00:00:00 2001 From: tangkong Date: Thu, 11 Jul 2024 11:46:25 -0700 Subject: [PATCH 2/8] MNT: Add shims kwarg for specifying shims in ControlLayer --- superscore/control_layers/core.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/superscore/control_layers/core.py b/superscore/control_layers/core.py index 1fda04b..da7835d 100644 --- a/superscore/control_layers/core.py +++ b/superscore/control_layers/core.py @@ -4,13 +4,18 @@ """ import asyncio from functools import singledispatchmethod -from typing import Any, Callable, Dict, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Union from superscore.control_layers.status import TaskStatus from ._aioca import AiocaShim from ._base_shim import _BaseShim +# available communication shim layers +SHIMS = { + 'ca': AiocaShim() +} + class ControlLayer: """ @@ -19,10 +24,13 @@ class ControlLayer: """ shims: Dict[str, _BaseShim] - def __init__(self, *args, **kwargs): - self.shims = { - 'ca': AiocaShim(), - } + def __init__(self, *args, shims: Optional[List[str]] = None, **kwargs): + if shims is None: + # load all available shims + self.shims = SHIMS + else: + self.shims = {key: shim for key, shim in SHIMS.items() if key in shims} + print(self.shims) def shim_from_pv(self, address: str) -> _BaseShim: """ From fffa86fa215b25cdb8abf2212de875e9d6c13776 Mon Sep 17 00:00:00 2001 From: tangkong Date: Thu, 11 Jul 2024 11:48:26 -0700 Subject: [PATCH 3/8] ENH: add Client configuration file loading and discovery, modify Client init to take control_layer and backend optionally --- superscore/client.py | 75 +++++++++++++++++++++++++++++++-- superscore/tests/config.cfg | 7 +++ superscore/tests/test_client.py | 39 +++++++++++++++++ 3 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 superscore/tests/config.cfg diff --git a/superscore/client.py b/superscore/client.py index f306aee..019247d 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -1,8 +1,12 @@ """Client for superscore. Used for programmatic interactions with superscore""" +import configparser import logging +import os +from pathlib import Path from typing import Any, Generator, List, Optional, Union from uuid import UUID +from superscore.backends import BACKENDS from superscore.backends.core import _Backend from superscore.control_layers import ControlLayer from superscore.control_layers.status import TaskStatus @@ -15,13 +19,76 @@ class Client: backend: _Backend cl: ControlLayer - def __init__(self, backend: _Backend, **kwargs) -> None: + def __init__( + self, + backend: Optional[_Backend] = None, + control_layer: Optional[ControlLayer] = None, + ) -> None: + if backend is None: + # set up a temp backend with temp file + backend = BACKENDS['test'] + if control_layer is None: + control_layer = ControlLayer() + self.backend = backend - self.cl = ControlLayer() + self.cl = control_layer @classmethod - def from_config(cls, cfg=None): - raise NotImplementedError + def from_config(cls, cfg: Path = None): + if not cfg: + cfg = cls.find_config() + if not os.path.exists(cfg): + raise RuntimeError(f"Superscore configuration file not found: {cfg}") + + cfg_parser = configparser.ConfigParser() + cfg_file = cfg_parser.read(cfg) + logger.debug(f"Loading configuration file at ({cfg_file})") + + # Gather Backend + if 'backend' in cfg_parser.sections(): + backend_type = cfg_parser.get("backend", "type") + kwargs = {key: value for key, value + in cfg_parser["backend"].items() + if key != "type"} + backend = BACKENDS[backend_type](**kwargs) + else: + backend = BACKENDS['test']() + + # configure control layer and shims + if 'control_layer' in cfg_parser.sections(): + shim_choices = [val for val, enabled + in cfg_parser["control_layer"].items() + if enabled] + print(shim_choices) + control_layer = ControlLayer(shims=shim_choices) + else: + control_layer = ControlLayer() + + return cls(backend=backend, control_layer=control_layer) + + @staticmethod + def find_config() -> Path: + """search the locations and stuff""" + # Point to with an environment variable + if os.environ.get('SUPERSCORE_CFG', False): + happi_cfg = os.environ.get('SUPERSCORE_CFG') + logger.debug("Found $SUPERSCORE_CFG specification for Client " + "configuration at %s", happi_cfg) + return happi_cfg + # Search in the current directory and home directory + else: + config_dirs = [os.environ.get('XDG_CONFIG_HOME', "."), + os.path.expanduser('~/.config'),] + for directory in config_dirs: + logger.debug('Searching for SuperScore config in %s', directory) + for path in ('.superscore.cfg', 'superscore.cfg'): + full_path = os.path.join(directory, path) + + if os.path.exists(full_path): + logger.debug("Found configuration file at %r", full_path) + return full_path + # If found nothing + raise OSError("No SuperScore configuration file found. Check SUPERSCORE_CFG.") def search(self, **post) -> Generator[Entry, None, None]: """Search by key-value pair. Can search by any field, including id""" diff --git a/superscore/tests/config.cfg b/superscore/tests/config.cfg new file mode 100644 index 0000000..aeff4f1 --- /dev/null +++ b/superscore/tests/config.cfg @@ -0,0 +1,7 @@ +[backend] +type = filestore +path = ./db/filestore.json + +[control_layer] +ca = true +pva = true diff --git a/superscore/tests/test_client.py b/superscore/tests/test_client.py index 3152d7a..785bac1 100644 --- a/superscore/tests/test_client.py +++ b/superscore/tests/test_client.py @@ -1,10 +1,43 @@ +import os +from pathlib import Path from unittest.mock import patch +import pytest + +from superscore.backends.filestore import FilestoreBackend from superscore.client import Client from superscore.model import Root from .conftest import MockTaskStatus +SAMPLE_CFG = Path(__file__).parent / 'config.cfg' + + +@pytest.fixture(scope='function') +def xdg_config_patch(tmp_path): + config_home = tmp_path / 'xdg_config_home' + config_home.mkdir() + return config_home + + +@pytest.fixture(scope='function') +def sscore_cfg(xdg_config_patch: Path): + # patch config discovery paths + xdg_cfg = os.environ.get("XDG_CONFIG_HOME", '') + sscore_cfg = os.environ.get("SUPERSCORE_CFG", '') + + os.environ['XDG_CONFIG_HOME'] = str(xdg_config_patch) + os.environ['SUPERSCORE_CFG'] = '' + + sscore_cfg_path = xdg_config_patch / "superscore.cfg" + sscore_cfg_path.symlink_to(SAMPLE_CFG) + + yield str(sscore_cfg_path) + + # reset env vars + os.environ["SUPERSCORE_CFG"] = str(sscore_cfg) + os.environ["XDG_CONFIG_HOME"] = xdg_cfg + @patch('superscore.control_layers.core.ControlLayer.put') def test_apply(put_mock, mock_client: Client, sample_database: Root): @@ -19,3 +52,9 @@ def test_apply(put_mock, mock_client: Client, sample_database: Root): mock_client.apply(snap, sequential=True) assert put_mock.call_count == 3 + + +def test_from_cfg(sscore_cfg: str): + client = Client.from_config() + assert isinstance(client.backend, FilestoreBackend) + assert 'ca' in client.cl.shims From 0f9689d95a8492643f98534e78e889adec723a7f Mon Sep 17 00:00:00 2001 From: tangkong Date: Thu, 11 Jul 2024 13:53:19 -0700 Subject: [PATCH 4/8] DOC: improve docstrings --- superscore/client.py | 54 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/superscore/client.py b/superscore/client.py index 019247d..5ae90b0 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -35,6 +35,39 @@ def __init__( @classmethod def from_config(cls, cfg: Path = None): + """ + Create a client from the configuration file specification. + + Configuration file should be of an "ini" format, along the lines of: + + .. code:: + + [backend] + type = filestore + path = ./db/filestore.json + + [control_layer] + ca = true + pva = true + + The ``backend`` section has one special key ("type"), and the rest of the + settings are passed to the appropriate ``_Backend`` as keyword arguments. + + The ``control_layer`` section has a key-value pair for each available shim. + The ``ControlLayer`` object will be created with all the valid shims with + True values. + + Parameters + ---------- + cfg : Path, optional + Path to a configuration file, by default None. If omitted, + :meth:`.find_config` will be used to find one + + Raises + ------ + RuntimeError + If a configuration file cannot be found + """ if not cfg: cfg = cls.find_config() if not os.path.exists(cfg): @@ -68,7 +101,22 @@ def from_config(cls, cfg: Path = None): @staticmethod def find_config() -> Path: - """search the locations and stuff""" + """ + Search for a ``superscore`` configuation file. Searches in the following + locations in order for a file named "superscore.cfg": + - environment variable ``$SUPERSCORE_CFG`` (a full path) + - folder set in environment variable ``$XDG_CONFIG_HOME`` + + Returns + ------- + path : str + Absolute path to the configuration file + + Raises + ------ + OSError + If no configuration file can be found by the described methodology + """ # Point to with an environment variable if os.environ.get('SUPERSCORE_CFG', False): happi_cfg = os.environ.get('SUPERSCORE_CFG') @@ -80,7 +128,7 @@ def find_config() -> Path: config_dirs = [os.environ.get('XDG_CONFIG_HOME', "."), os.path.expanduser('~/.config'),] for directory in config_dirs: - logger.debug('Searching for SuperScore config in %s', directory) + logger.debug('Searching for superscore config in %s', directory) for path in ('.superscore.cfg', 'superscore.cfg'): full_path = os.path.join(directory, path) @@ -88,7 +136,7 @@ def find_config() -> Path: logger.debug("Found configuration file at %r", full_path) return full_path # If found nothing - raise OSError("No SuperScore configuration file found. Check SUPERSCORE_CFG.") + raise OSError("No superscore configuration file found. Check SUPERSCORE_CFG.") def search(self, **post) -> Generator[Entry, None, None]: """Search by key-value pair. Can search by any field, including id""" From b828cbd0cfddca4e7da8d2e979f20ccfba90ea21 Mon Sep 17 00:00:00 2001 From: tangkong Date: Thu, 11 Jul 2024 13:53:39 -0700 Subject: [PATCH 5/8] TST: add one more test --- superscore/tests/test_client.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/superscore/tests/test_client.py b/superscore/tests/test_client.py index 785bac1..bcd293a 100644 --- a/superscore/tests/test_client.py +++ b/superscore/tests/test_client.py @@ -58,3 +58,11 @@ def test_from_cfg(sscore_cfg: str): client = Client.from_config() assert isinstance(client.backend, FilestoreBackend) assert 'ca' in client.cl.shims + + +def test_find_config(sscore_cfg: str): + assert sscore_cfg == Client.find_config() + + # explicit SUPERSCORE_CFG env var supercedes XDG_CONFIG_HOME + os.environ['SUPERSCORE_CFG'] = 'other/cfg' + assert 'other/cfg' == Client.find_config() From 8b81ac4b7bd41bae3832c43f95dbcfebaa851936 Mon Sep 17 00:00:00 2001 From: tangkong Date: Thu, 11 Jul 2024 14:48:49 -0700 Subject: [PATCH 6/8] DOC: pre-release notes --- .../54-enh_config_client.rst | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/source/upcoming_release_notes/54-enh_config_client.rst diff --git a/docs/source/upcoming_release_notes/54-enh_config_client.rst b/docs/source/upcoming_release_notes/54-enh_config_client.rst new file mode 100644 index 0000000..3ee2fe7 --- /dev/null +++ b/docs/source/upcoming_release_notes/54-enh_config_client.rst @@ -0,0 +1,22 @@ +54 enh_config_client +#################### + +API Breaks +---------- +- N/A + +Features +-------- +- Adds ability for Client to search for configuration files, and load settings from them. + +Bugfixes +-------- +- N/A + +Maintenance +----------- +- N/A + +Contributors +------------ +- tangkong From b6076eced0085be2dcc5a70eec2d3e37cf94f2ad Mon Sep 17 00:00:00 2001 From: tangkong Date: Fri, 12 Jul 2024 14:17:26 -0700 Subject: [PATCH 7/8] DOC: clean up docstrings --- superscore/client.py | 8 ++++---- superscore/control_layers/core.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/superscore/client.py b/superscore/client.py index 5ae90b0..0b4c21a 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -92,7 +92,6 @@ def from_config(cls, cfg: Path = None): shim_choices = [val for val, enabled in cfg_parser["control_layer"].items() if enabled] - print(shim_choices) control_layer = ControlLayer(shims=shim_choices) else: control_layer = ControlLayer() @@ -103,9 +102,10 @@ def from_config(cls, cfg: Path = None): def find_config() -> Path: """ Search for a ``superscore`` configuation file. Searches in the following - locations in order for a file named "superscore.cfg": - - environment variable ``$SUPERSCORE_CFG`` (a full path) - - folder set in environment variable ``$XDG_CONFIG_HOME`` + locations in order + - ``$SUPERSCORE_CFG`` (a full path to a config file) + - ``$XDG_CONFIG_HOME/{superscore.cfg, .superscore.cfg}`` (either filename) + - ``~/.config/{superscore.cfg, .superscore.cfg}`` Returns ------- diff --git a/superscore/control_layers/core.py b/superscore/control_layers/core.py index da7835d..3e7b801 100644 --- a/superscore/control_layers/core.py +++ b/superscore/control_layers/core.py @@ -30,7 +30,6 @@ def __init__(self, *args, shims: Optional[List[str]] = None, **kwargs): self.shims = SHIMS else: self.shims = {key: shim for key, shim in SHIMS.items() if key in shims} - print(self.shims) def shim_from_pv(self, address: str) -> _BaseShim: """ From 251b62b48a40d17c7063d329f066995648851717 Mon Sep 17 00:00:00 2001 From: tangkong Date: Mon, 15 Jul 2024 14:03:34 -0700 Subject: [PATCH 8/8] MNT: refactor to use get_backend factory function, adjust TestBackend to accept no arguments --- superscore/backends/__init__.py | 24 ++++++++++++++++++++++-- superscore/backends/test.py | 9 ++++++--- superscore/client.py | 18 +++++++++++------- superscore/control_layers/core.py | 7 +++++++ 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/superscore/backends/__init__.py b/superscore/backends/__init__.py index 4d5a579..4fe5662 100644 --- a/superscore/backends/__init__.py +++ b/superscore/backends/__init__.py @@ -8,6 +8,9 @@ logger = logging.getLogger(__name__) +BACKENDS: Dict[str, _Backend] = {} + + def _get_backend(backend: str) -> _Backend: if backend == 'filestore': from .filestore import FilestoreBackend @@ -19,7 +22,7 @@ def _get_backend(backend: str) -> _Backend: raise ValueError(f"Unknown backend {backend}") -def _get_backends() -> Dict[str, _Backend]: +def _init_backends() -> Dict[str, _Backend]: backends = {} try: @@ -35,4 +38,21 @@ def _get_backends() -> Dict[str, _Backend]: return backends -BACKENDS = _get_backends() +def get_backend(backend_name: str) -> _Backend: + try: + backend = BACKENDS[backend_name] + except KeyError: + # try to load it + try: + BACKENDS[backend_name] = _get_backend(backend_name) + backend = BACKENDS[backend_name] + except ValueError: + raise ValueError(f'Backend {backend_name} not supported. Available ' + f'backends include: ({list(BACKENDS.keys())})') + except ImportError as ex: + raise ValueError(f'Backend {(backend_name)} failed to load: {ex}') + + return backend + + +BACKENDS = _init_backends() diff --git a/superscore/backends/test.py b/superscore/backends/test.py index f9ae603..0975327 100644 --- a/superscore/backends/test.py +++ b/superscore/backends/test.py @@ -1,7 +1,7 @@ """ Backend that manipulates Entries in-memory for testing purposes. """ -from typing import List +from typing import List, Optional from uuid import UUID from superscore.backends.core import _Backend @@ -12,8 +12,11 @@ class TestBackend(_Backend): """Backend that manipulates Entries in-memory, for testing purposes.""" - def __init__(self, data: List[Entry]): - self.data = data + def __init__(self, data: Optional[List[Entry]] = None): + if data is None: + self.data = [] + else: + self.data = data def save_entry(self, entry: Entry) -> None: try: diff --git a/superscore/client.py b/superscore/client.py index 0b4c21a..89c4a40 100644 --- a/superscore/client.py +++ b/superscore/client.py @@ -6,7 +6,7 @@ from typing import Any, Generator, List, Optional, Union from uuid import UUID -from superscore.backends import BACKENDS +from superscore.backends import get_backend from superscore.backends.core import _Backend from superscore.control_layers import ControlLayer from superscore.control_layers.status import TaskStatus @@ -26,7 +26,8 @@ def __init__( ) -> None: if backend is None: # set up a temp backend with temp file - backend = BACKENDS['test'] + logger.warning('No backend specified, loading an empty test backend') + backend = get_backend('test')() if control_layer is None: control_layer = ControlLayer() @@ -83,9 +84,11 @@ def from_config(cls, cfg: Path = None): kwargs = {key: value for key, value in cfg_parser["backend"].items() if key != "type"} - backend = BACKENDS[backend_type](**kwargs) + backend_class = get_backend(backend_type) + backend = backend_class(**kwargs) else: - backend = BACKENDS['test']() + logger.warning('No backend specified, loading an empty test backend') + backend = get_backend('test')() # configure control layer and shims if 'control_layer' in cfg_parser.sections(): @@ -94,6 +97,7 @@ def from_config(cls, cfg: Path = None): if enabled] control_layer = ControlLayer(shims=shim_choices) else: + logger.debug('No control layer shims specified, loading all available') control_layer = ControlLayer() return cls(backend=backend, control_layer=control_layer) @@ -119,10 +123,10 @@ def find_config() -> Path: """ # Point to with an environment variable if os.environ.get('SUPERSCORE_CFG', False): - happi_cfg = os.environ.get('SUPERSCORE_CFG') + superscore_cfg = os.environ.get('SUPERSCORE_CFG') logger.debug("Found $SUPERSCORE_CFG specification for Client " - "configuration at %s", happi_cfg) - return happi_cfg + "configuration at %s", superscore_cfg) + return superscore_cfg # Search in the current directory and home directory else: config_dirs = [os.environ.get('XDG_CONFIG_HOME', "."), diff --git a/superscore/control_layers/core.py b/superscore/control_layers/core.py index 3e7b801..889b372 100644 --- a/superscore/control_layers/core.py +++ b/superscore/control_layers/core.py @@ -3,6 +3,7 @@ and dispatches to various shims depending on the context. """ import asyncio +import logging from functools import singledispatchmethod from typing import Any, Callable, Dict, List, Optional, Union @@ -11,6 +12,8 @@ from ._aioca import AiocaShim from ._base_shim import _BaseShim +logger = logging.getLogger(__name__) + # available communication shim layers SHIMS = { 'ca': AiocaShim() @@ -28,8 +31,12 @@ def __init__(self, *args, shims: Optional[List[str]] = None, **kwargs): if shims is None: # load all available shims self.shims = SHIMS + logger.debug('No shims specified, loading all available communication ' + f'shims: {list(self.shims.keys())}') else: self.shims = {key: shim for key, shim in SHIMS.items() if key in shims} + logger.debug('Loaded valid shims from the requested list: ' + f'{list(self.shims.keys())}') def shim_from_pv(self, address: str) -> _BaseShim: """