Use GraphQL client instead of custom model and IO (#9)

This uses the GraphQL client defined in the `cryoet-data-portal` package
related to this project. The main motivations for doing this are as
follows.

1. Reduce the code to maintain here.
2. Simplify keeping up with structural/schematic changes to the data
portal.
3. Exercise the code in `cryoet-data-portal` and improve it.

Listing the datasets and tomograms seems a little slower than using
s3fs/fsspec, but the benefits are worth the cost.

This removes support for the local filesystem as a storage backend for
the portal data because obviously the client does not support that. The
code to support different URLs for the GraphQL entry point are kept, but
we make the line-edit read-only in the GUI because only the default URL
works and others cause problems.

This also removes mocks from some of the tests because mocking the
client is trickier than mocking our own classes/functions. Tests take
longer, but we adjusted the timeouts to account for that. That's good
enough as a short term solution, but ultimately we need mocks/fakes to
avoid long running tests.

setup.cfg

@@ -31,12 +31,13 @@ project_urls =
 [options]
 packages = find:
 install_requires =
+    cryoet_data_portal
+    fsspec[http,s3]
     npe2
     numpy
     napari_ome_zarr
     ndjson
     qtpy
-    s3fs
     superqt
 
 python_requires = >=3.8

src/napari_cryoet_data_portal/__init__.py

@@ -2,10 +2,11 @@
     from ._version import version as __version__
 except ImportError:
     __version__ = "unknown"
-from ._logging import logger
 from ._reader import (
     points_annotations_reader,
-    read_points_annotations_json,
+    read_annotation,
+    read_points_annotations_ndjson,
+    read_tomogram,
     read_tomogram_ome_zarr,
     tomogram_ome_zarr_reader,
 )
@@ -14,7 +15,9 @@
 __all__ = (
     "DataPortalWidget",
     "points_annotations_reader",
+    "read_annotation",
+    "read_tomogram",
     "read_tomogram_ome_zarr",
-    "read_points_annotations_json",
+    "read_points_annotations_ndjson",
     "tomogram_ome_zarr_reader",
 )

src/napari_cryoet_data_portal/_listing_widget.py

@@ -1,4 +1,4 @@
-from typing import Generator, Optional
+from typing import Generator, List, Optional, Tuple
 
 from qtpy.QtCore import Qt
 from qtpy.QtWidgets import (
@@ -8,11 +8,10 @@
     QVBoxLayout,
     QWidget,
 )
+from cryoet_data_portal import Client, Dataset, Tomogram
 
-from napari_cryoet_data_portal._io import list_dir
 from napari_cryoet_data_portal._listing_tree_widget import ListingTreeWidget
 from napari_cryoet_data_portal._logging import logger
-from napari_cryoet_data_portal._model import Dataset
 from napari_cryoet_data_portal._progress_widget import ProgressWidget
 from napari_cryoet_data_portal._vendored.superqt._searchable_tree_widget import (
     _update_visible_items,
@@ -44,37 +43,37 @@ def __init__(self, parent: Optional[QWidget] = None) -> None:
         layout.addStretch(0)
         self.setLayout(layout)
 
-    def load(self, path: str) -> None:
-        """Lists the datasets and tomograms at the given data portal path."""
-        logger.debug("ListingWidget.load: %s", path)
+    def load(self, uri: str) -> None:
+        """Lists the datasets and tomograms using the given portal URI."""
+        logger.debug("ListingWidget.load: %s", uri)
         self.tree.clear()
         self.show()
-        self._progress.submit(path)
+        self._progress.submit(uri)
 
     def cancel(self) -> None:
         """Cancels the last listing."""
         logger.debug("ListingWidget.cancel")
         self._progress.cancel()
 
-    def _loadDatasets(self, path: str) -> Generator[Dataset, None, None]:
-        logger.debug("ListingWidget._loadDatasets: %s", path)
-        # TODO: only list non-hidden directories.
-        dataset_names = tuple(
-            p for p in list_dir(path) if not p.startswith(".")
-        )
-        if len(dataset_names) == 0:
-            logger.debug("No datasets found")
-        for name in dataset_names:
-            yield Dataset.from_data_path_and_name(path, name)
+    def _loadDatasets(self, uri: str) -> Generator[Tuple[Dataset, List[Tomogram]], None, None]:
+        logger.debug("ListingWidget._loadDatasets: %s", uri)
+        client = Client(uri)
+        for dataset in Dataset.find(client):
+            tomograms: List[Tomogram] = []
+            for run in dataset.runs:
+                for spacing in run.tomogram_voxel_spacings:
+                    tomograms.extend(spacing.tomograms)
+            yield dataset, tomograms
 
-    def _onDatasetLoaded(self, dataset: Dataset) -> None:
-        logger.debug("ListingWidget._onDatasetLoaded: %s", dataset.name)
-        text = f"{dataset.name} ({len(dataset.tomograms)})"
+    def _onDatasetLoaded(self, result: Tuple[Dataset, List[Tomogram]]) -> None:
+        dataset, tomograms = result
+        logger.debug("ListingWidget._onDatasetLoaded: %s", dataset.id)
+        text = f"{dataset.id} ({len(tomograms)})"
         item = QTreeWidgetItem((text,))
         item.setData(0, Qt.ItemDataRole.UserRole, dataset)
-        for s in dataset.tomograms:
-            tomogram_item = QTreeWidgetItem((s.name,))
-            tomogram_item.setData(0, Qt.ItemDataRole.UserRole, s)
+        for tomogram in tomograms:
+            tomogram_item = QTreeWidgetItem((tomogram.name,))
+            tomogram_item.setData(0, Qt.ItemDataRole.UserRole, tomogram)
             item.addChild(tomogram_item)
         _update_visible_items(item, self.tree.last_filter)
         self.tree.addTopLevelItem(item)

src/napari_cryoet_data_portal/_metadata_widget.py

@@ -5,10 +5,9 @@
     QVBoxLayout,
     QWidget,
 )
+from cryoet_data_portal import Dataset, Tomogram
 
-from napari_cryoet_data_portal._io import read_json
 from napari_cryoet_data_portal._logging import logger
-from napari_cryoet_data_portal._model import Dataset, Tomogram
 from napari_cryoet_data_portal._progress_widget import ProgressWidget
 from napari_cryoet_data_portal._vendored.superqt._searchable_tree_widget import (
     QSearchableTreeWidget,
@@ -39,7 +38,8 @@ def load(self, data: Union[Dataset, Tomogram]) -> None:
         """Loads the JSON metadata of the given dataset or tomogram."""
         logger.debug("MetadataWidget.load: %s", data)
         self._main.tree.clear()
-        self.setTitle(f"Metadata: {data.name}")
+        name = data.id if isinstance(data, Dataset) else data.name
+        self.setTitle(f"Metadata: {name}")
         self.show()
         self._progress.submit(data)
 
@@ -50,14 +50,7 @@ def cancel(self) -> None:
 
     def _loadMetadata(self, data: Union[Dataset, Tomogram]) -> Dict[str, Any]:
         logger.debug("MetadataWidget._loadMetadata: %s", data)
-        path = ""
-        if isinstance(data, Dataset):
-            path = data.metadata_path
-        elif isinstance(data, Tomogram):
-            path = data.tomogram_metadata_path
-        else:
-            raise AssertionError("Expected Dataset or Tomogram data")
-        return read_json(path)
+        return data.to_dict()
 
     def _onMetadataLoaded(self, metadata: Dict[str, Any]) -> None:
         logger.debug("MetadataWidget._onMetadataLoaded: %s", metadata)