add arim.datasets, to automatically fetch example datasets

examples/contact_tfm.py

@@ -10,6 +10,7 @@
 import numpy as np
 
 import arim
+import arim.datasets
 import arim.im.tfm
 import arim.io
 import arim.plot as aplt
@@ -21,7 +22,7 @@
 
 # %% Load datafile (exported from BRAIN)
 
-expdata_filename = r"example-datasets/contact_notch_aluminium.mat"
+expdata_filename = arim.datasets.EXAMPLES.fetch("contact_notch_aluminium.mat")
 frame = arim.io.load_expdata(expdata_filename)
 
 print("Frame:")

examples/multiview_tfm_contact/conf.yaml

@@ -1,5 +1,6 @@
 frame:
-  datafile: ../example-datasets/contact_notch_aluminium.mat
+  dataset_name: examples
+  dataset_item: contact_notch_aluminium.mat
   instrument_delay: 0. # shift the t=0 defined in datafile by this number.
 
 probe:

examples/multiview_tfm_immersion/conf.yaml

@@ -1,5 +1,6 @@
 frame:
-  datafile: ../example-datasets/immersion_notch_aluminium.mat
+  dataset_name: examples
+  dataset_item: immersion_notch_aluminium.mat
   instrument_delay: 500.e-9 # shift the t=0 defined in datafile by this number.
 
 probe:

pyproject.toml

@@ -31,7 +31,8 @@ dependencies = [
   "numpy",
   "scipy",
   "numba",
-  "pyyaml"
+  "pyyaml",
+  "pooch",
 ]
 
 [project.optional-dependencies]

src/arim/datasets.py

@@ -0,0 +1,25 @@
+"""
+Example datasets
+
+Usage::
+
+    fname = arim.datasets.EXAMPLES.fetch("contact_notch_aluminium.mat")
+
+
+"""
+
+import pooch
+
+EXAMPLES = pooch.create(
+    path=pooch.os_cache("arim"),
+    base_url="https://github.com/ndtatbristol/arim/raw/25391358a02d121c6fe0f3114e193606e581c8b9/examples/example-datasets/",
+    version=None,
+    version_dev=None,
+    registry={
+        "contact_notch_aluminium.mat": "sha256:c58d25ccaf3081348c392d98a45a5e28809c80c900782a9b60f62d1a7ef2c76f",
+        "immersion_notch_aluminium.mat": "sha256:ae125a6c461a10ded54ca19e223c4d39fe857be8c12fda6f97b61de25a52a08e",
+    },
+)
+
+# Dictionary of all datasets
+DATASETS: dict[str, pooch.Pooch] = {"examples": EXAMPLES}

src/arim/io/native.py

@@ -22,17 +22,32 @@
 If a configuration entry is present in two files, only the entry from the file
 read the latest will be kept.
 
+
+conf.yaml format
+----------------
+
+The ultrasonic data is provided using either:
+
+1. ``frame.datafile``: path to the data file, either absolute or relative to
+   ``conf.yaml``,
+2. ``frame.dataset_name`` and ``frame.dataset_item``: a dataset name
+   (``arim.datasets``) and the path of the item to fetch
+
+
 """
 
 import copy
+import logging
 import pathlib
 
 import numpy as np
 import yaml
 
-from .. import _probes, config, core, geometry
+from .. import _probes, config, core, datasets, geometry
 from . import brain
 
+LOGGER = logger = logging.getLogger(__name__)
+
 __all__ = [
     "load_conf",
     "load_conf_file",
@@ -379,7 +394,25 @@ def frame_from_conf(
 
     """
     frame_conf = conf["frame"]
-    frame = brain.load_expdata(frame_conf["datafile"])
+    if "datafile" in frame_conf:
+        # Load from absolute or relative path:
+        fname = frame_conf["datafile"]
+        if "dataset_name" in frame_conf:
+            LOGGER.warning("ignoring frame.dataset_name")
+        if "dataset_item" in frame_conf:
+            LOGGER.warning("ignoring frame.dataset_item")
+    else:
+        # Load from arim.datasets:
+        dataset_name = frame_conf["dataset_name"]
+        try:
+            dataset_pooch = datasets.DATASETS[dataset_name]
+        except (KeyError, TypeError):
+            raise ValueError(
+                f"Unknown dataset: '{dataset_name}'. Valid values are: {tuple(datasets.DATASETS.keys())}"
+            )
+        fname = dataset_pooch.fetch(frame_conf["dataset_item"])
+
+    frame = brain.load_expdata(fname)
 
     instrument_delay = None
     try:

tests/io/test_native.py

@@ -1,3 +1,4 @@
+import arim.datasets
 import arim.io
 
 PROBE_1 = """
@@ -90,6 +91,13 @@
   z: 40.e-3
 """
 
+FRAME_FROM_DSET = """
+frame:
+  dataset_name: examples
+  dataset_item: contact_notch_aluminium.mat
+  instrument_delay: 0.
+"""
+
 
 def test_probe_from_conf():
     for conf_str in (PROBE_1, PROBE_2):
@@ -120,3 +128,22 @@ def test_material_attenuation_from_conf():
         conf = arim.io.load_conf_from_str(conf_str)
         att = arim.io.material_attenuation_from_conf(conf["longitudinal_att"])
         assert att(10) == 777.0
+
+
+def test_frame_from_conf():
+    # Load frame using arim.datasets:
+    conf = arim.io.load_conf_from_str(FRAME_FROM_DSET)
+    frame = arim.io.frame_from_conf(
+        conf, use_probe_from_conf=False, use_examination_object_from_conf=False
+    )
+    assert frame.numtimetraces == (64 * 65) // 2
+
+
+def test_frame_from_conf2():
+    # Load frame using absolute path:
+    fname = arim.datasets.EXAMPLES.fetch("contact_notch_aluminium.mat")
+    conf = {"frame": {"datafile": str(fname)}}
+    frame = arim.io.frame_from_conf(
+        conf, use_probe_from_conf=False, use_examination_object_from_conf=False
+    )
+    assert frame.numtimetraces == (64 * 65) // 2