Add two records: CreateDirectories and DirectoryExists. Automatically create directories based on this PV like in AD

docs/how-to/capture-hdf.md

@@ -9,6 +9,8 @@ These can be viewed from the DATA screen.
 ```
 
 - The file directory and name are chosen with `:DATA:HDFDirectory` and `:DATA:HDFFileName`.
+- The number of directories that the IOC is allowed to create provided they don't exist is determined by `:DATA:CreateDirectory`. The behavior of this signal is the same as the identical PV in [`areaDetector`](https://areadetector.github.io/areaDetector/ADCore/NDPluginFile.html).
+- `DATA:DirectoryExists` represents whether or not the directory specified exists and is writable by the user under which the IOC is running.
 - `:DATA:NumCapture` is the number of frames to capture in the file.
 - `:DATA:NumCaptured` is the number of frames written to file.
 - `:DATA:NumReceived` is the number of frames received from the panda.

src/pandablocks_ioc/_hdf_ioc.py

@@ -5,6 +5,7 @@
 from collections import deque
 from enum import Enum
 from importlib.util import find_spec
+from pathlib import Path
 from typing import Callable, Deque, Optional, Union
 
 from pandablocks.asyncio import AsyncioClient
@@ -311,6 +312,8 @@
     _client: AsyncioClient
 
     _directory_record: RecordWrapper
+    _create_directory_record: RecordWrapper
+    _directory_exists_record: RecordWrapper
     _file_name_record: RecordWrapper
     _file_number_record: RecordWrapper
     _file_format_record: RecordWrapper
@@ -340,7 +343,8 @@
             length=path_length,
             DESC="File path for HDF5 files",
             validate=self._parameter_validate,
-            on_update=self._update_full_file_path,
+            on_update=self._update_directory_path,
+            always_update=True,
         )
         add_automatic_pvi_info(
             PviGroup.HDF,
@@ -352,6 +356,40 @@
             record_prefix + ":" + self._DATA_PREFIX + ":HDFDirectory"
         )
 
+        create_directory_record_name = EpicsName(self._DATA_PREFIX + ":CreateDirectory")
+        self._create_directory_record = builder.longOut(
+            create_directory_record_name,
+            initial_value=0,
+            DESC="Directory creation depth",
+        )
+        add_automatic_pvi_info(
+            PviGroup.HDF,
+            self._create_directory_record,
+            create_directory_record_name,
+            builder.longOut,
+        )
+        self._create_directory_record.add_alias(
+            record_prefix + ":" + create_directory_record_name.upper()
+        )
+
+        directory_exists_name = EpicsName(self._DATA_PREFIX + ":DirectoryExists")
+        self._directory_exists_record = builder.boolIn(
+            directory_exists_name,
+            ZNAM="No",
+            ONAM="Yes",
+            initial_value=0,
+            DESC="Directory exists",
+        )
+        add_automatic_pvi_info(
+            PviGroup.HDF,
+            self._directory_exists_record,
+            directory_exists_name,
+            builder.boolIn,
+        )
+        self._directory_exists_record.add_alias(
+            record_prefix + ":" + directory_exists_name.upper()
+        )
+
         file_name_record_name = EpicsName(self._DATA_PREFIX + ":HDF_FILE_NAME")
         self._file_name_record = builder.longStringOut(
             file_name_record_name,
@@ -523,6 +561,70 @@
             return False
         return True
 
+    async def _update_directory_path(self, new_val) -> None:
+        """Handles writes to the directory path PV, creating
+        directories based on the setting of the CreateDirectory record"""
+        new_path = Path(new_val).absolute()
+        create_dir_depth = self._create_directory_record.get()
+        max_dirs_to_create = 0
+        if create_dir_depth < 0:
+            max_dirs_to_create = abs(create_dir_depth)
+        elif create_dir_depth > len(new_path.parents):
+            max_dirs_to_create = 0
+        elif create_dir_depth > 0:
+            max_dirs_to_create = len(new_path.parents) - create_dir_depth
+
+        logging.debug(f"Permitted to create up to {max_dirs_to_create} dirs.")
+        dirs_to_create = 0
+        for p in reversed(new_path.parents):
+            if not p.exists():
+                if dirs_to_create == 0:
+                    # First directory level that does not exist, log it.
+                    logging.error(f"All dir from {str(p)} and below do not exist!")
+                dirs_to_create += 1
+            else:
+                logging.info(f"{str(p)} exists")
+
+        # Account for target path itself not existing
+        if not os.path.exists(new_path):
+            dirs_to_create += 1
+
+        logging.debug(f"Need to create {dirs_to_create} directories.")
+
+        # Case where all dirs exist
+        if dirs_to_create == 0:
+            if os.access(new_path, os.W_OK):
+                status_msg = "Dir exists and is writable"
+                self._directory_exists_record.set(1)
+            else:
+                status_msg = "Dirs exist but aren't writable."
+                self._directory_exists_record.set(0)
+        # Case where we will create directories
+        elif dirs_to_create <= max_dirs_to_create:
+            logging.debug(f"Attempting to create {dirs_to_create} dir(s)...")
+            try:
+                os.makedirs(new_path, exist_ok=True)
+                status_msg = f"Created {dirs_to_create} dirs."
+                self._directory_exists_record.set(1)
+            except PermissionError:
+                status_msg = "Permission error creating dirs!"
+                self._directory_exists_record.set(0)
+        # Case where too many directories need to be created
+        else:
+            status_msg = f"Need to create {dirs_to_create} > {max_dirs_to_create} dirs."
+            self._directory_exists_record.set(0)
+
+        if self._directory_exists_record.get() == 0:
+            sevr = alarm.MAJOR_ALARM, alrm = alarm.STATE_ALARM
+            logging.error(status_msg)
+        else:
+            sevr = alarm.NO_ALARM, alrm = alarm.NO_ALARM
+            logging.debug(status_msg)
+
+        self._status_message_record.set(status_msg, severity=sevr, alarm=alrm)
+
+        await self._update_full_file_path(new_val)
+
     async def _update_full_file_path(self, new_val) -> None:
         self._full_file_path_record.set(self._get_filepath())
 
@@ -532,6 +634,12 @@
         This method expects to be run as an asyncio Task."""
         try:
             # Set up the hdf buffer
+
+            if not self._directory_exists_record.get() == 1:
+                raise RuntimeError(
+                    "Configured HDF directory does not exist or is not writable!"
+                )
+
             num_capture: int = self._num_capture_record.get()
             capture_mode: CaptureMode = CaptureMode(self._capture_mode_record.get())
             filepath = self._get_filepath()

tests/conftest.py

@@ -1,4 +1,23 @@
 # flake8: noqa
 
+import os
+
 from fixtures.mocked_panda import *
 from fixtures.panda_data import *
+
+
+# Autouse fixture that will set all EPICS networking env vars to use lo interface
+# to avoid false failures caused by things like firewalls blocking EPICS traffic.
+@pytest.fixture(scope="session", autouse=True)
+def configure_epics_environment():
+    os.environ["EPICS_CAS_INTF_ADDR_LIST"] = "127.0.0.1"
+    os.environ["EPICS_CAS_BEACON_ADDR_LIST"] = "127.0.0.1"
+    os.environ["EPICS_CA_ADDR_LIST"] = "127.0.0.1"
+    os.environ["EPICS_CAS_AUTO_ADDR_LIST"] = "NO"
+    os.environ["EPICS_CA_AUTO_BEACON_ADDR_LIST"] = "NO"
+
+    os.environ["EPICS_PVAS_INTF_ADDR_LIST"] = "127.0.0.1"
+    os.environ["EPICS_PVAS_BEACON_ADDR_LIST"] = "127.0.0.1"
+    os.environ["EPICS_PVA_ADDR_LIST"] = "127.0.0.1"
+    os.environ["EPICS_PVAS_AUTO_BEACON_ADDR_LIST"] = "NO"
+    os.environ["EPICS_PVA_AUTO_ADDR_LIST"] = "NO"

tests/test-bobfiles/DATA.bob

@@ -3,7 +3,7 @@
   <x>0</x>
   <y>0</y>
   <width>506</width>
-  <height>413</height>
+  <height>463</height>
   <grid_step_x>4</grid_step_x>
   <grid_step_y>4</grid_step_y>
   <widget type="label" version="2.0.0">
@@ -30,7 +30,7 @@
     <x>5</x>
     <y>30</y>
     <width>496</width>
-    <height>106</height>
+    <height>156</height>
     <transparent>true</transparent>
     <widget type="label" version="2.0.0">
       <name>Label</name>
@@ -52,35 +52,73 @@
     </widget>
     <widget type="label" version="2.0.0">
       <name>Label</name>
-      <text>Hdf File Name</text>
+      <text>Createdirectory</text>
       <x>0</x>
       <y>25</y>
       <width>250</width>
       <height>20</height>
     </widget>
     <widget type="textentry" version="3.0.0">
       <name>TextEntry</name>
-      <pv_name>TEST_PREFIX:DATA:HDF_FILE_NAME</pv_name>
+      <pv_name>TEST_PREFIX:DATA:CreateDirectory</pv_name>
       <x>255</x>
       <y>25</y>
       <width>205</width>
       <height>20</height>
       <horizontal_alignment>1</horizontal_alignment>
+    </widget>
+    <widget type="label" version="2.0.0">
+      <name>Label</name>
+      <text>Directoryexists</text>
+      <x>0</x>
+      <y>50</y>
+      <width>250</width>
+      <height>20</height>
+    </widget>
+    <widget type="textupdate" version="2.0.0">
+      <name>TextUpdate</name>
+      <pv_name>TEST_PREFIX:DATA:DirectoryExists</pv_name>
+      <x>255</x>
+      <y>50</y>
+      <width>205</width>
+      <height>20</height>
+      <font>
+        <font name="Default Bold" family="Liberation Sans" style="BOLD" size="14.0">
+      </font>
+      </font>
+      <horizontal_alignment>1</horizontal_alignment>
+    </widget>
+    <widget type="label" version="2.0.0">
+      <name>Label</name>
+      <text>Hdf File Name</text>
+      <x>0</x>
+      <y>75</y>
+      <width>250</width>
+      <height>20</height>
+    </widget>
+    <widget type="textentry" version="3.0.0">
+      <name>TextEntry</name>
+      <pv_name>TEST_PREFIX:DATA:HDF_FILE_NAME</pv_name>
+      <x>255</x>
+      <y>75</y>
+      <width>205</width>
+      <height>20</height>
+      <horizontal_alignment>1</horizontal_alignment>
       <format>6</format>
     </widget>
     <widget type="label" version="2.0.0">
       <name>Label</name>
       <text>Hdf Full File Path</text>
       <x>0</x>
-      <y>50</y>
+      <y>100</y>
       <width>250</width>
       <height>20</height>
     </widget>
     <widget type="textupdate" version="2.0.0">
       <name>TextUpdate</name>
       <pv_name>TEST_PREFIX:DATA:HDF_FULL_FILE_PATH</pv_name>
       <x>255</x>
-      <y>50</y>
+      <y>100</y>
       <width>205</width>
       <height>20</height>
       <font>
@@ -94,7 +132,7 @@
   <widget type="group" version="2.0.0">
     <name>CAPTURE</name>
     <x>5</x>
-    <y>141</y>
+    <y>191</y>
     <width>496</width>
     <height>206</height>
     <transparent>true</transparent>
@@ -268,7 +306,7 @@
   <widget type="group" version="2.0.0">
     <name>OUTPUTS</name>
     <x>5</x>
-    <y>352</y>
+    <y>402</y>
     <width>496</width>
     <height>56</height>
     <transparent>true</transparent>