Fix/use fsspec (#6)

* Use fsspec for io.

* Fix hardcoded usage of local file paths for href determination.

* delete test data, switch to using remote read with fsspec in tests and example, change arg name to make it clear were using remote hrefs

---------

Co-authored-by: emileten <[email protected]>

README.md

@@ -31,6 +31,8 @@ Command line functions
 
 ```shell
 stac nisarsim create-collection <destination/>
+
+stac nisarsim create-collection example-collection.json
 ```
 
 **Dither**: Indicates whether the data has been dithered or not:
@@ -53,7 +55,7 @@ stac nisarsim create-item <source/> <destination/> --dither <dither> --nmode <nm
 
 stac nisarsim create-item
 https://downloaduav.jpl.nasa.gov/Release2v/winnip_31604_12061_004_120717_L090_CX_07/
-examples/ --dither X --nmode 129
+example.json --dither X --nmode 129
 ```
 
 Use `stac nisarsim --help` to see all subcommands and options.

scripts/create_examples.py

@@ -11,14 +11,13 @@
 
 root = Path(__file__).parents[1]
 examples = root / "examples"
-nisar_sim_data = root / "tests" / "data-files"
 
 shutil.rmtree(examples, ignore_errors=True)
 
 stac_collection = stac.create_collection()
 
 item1 = stac.create_item(
-    str(nisar_sim_data / "winnip_31604_12061_004_120717_L090_CX_07/"),
+    "https://downloaduav.jpl.nasa.gov/Release2v/winnip_31604_12061_004_120717_L090_CX_07/",
     dither="X",
     nmode="129",
 )

src/stactools/nisar_sim/constants.py

@@ -55,3 +55,8 @@
 
 NMODE = ["129", "138", "143"]  # center frequency, bandwidth, and polarization
 XTALK = ["X", "C"]
+
+NISAR_SIM_EXAMPLE_HREF = (
+    "https://downloaduav.jpl.nasa.gov/Release2v/"
+    "winnip_31604_12061_004_120717_L090_CX_07/"
+)

src/stactools/nisar_sim/metadata.py

@@ -1,9 +1,9 @@
 import os
 import re
 from datetime import datetime
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple, TypeVar
+from typing import Any, Dict, List, Tuple, TypeVar
 
+import fsspec
 import h5py
 import pystac
 from pystac.extensions.sat import SatExtension
@@ -42,18 +42,14 @@ def _replace_dither(self, id: str, dither: str) -> str:
     def _href_modifier(
         self,
         href: str,
-        extension: Optional[str] = None,
-        nmode: Optional[str] = None,
+        extension: str,
+        nmode: str,
     ) -> str:
         _id = self._replace_dither(self.id, self.dither)
-        path = Path(href) / _id
-
-        if extension:
-            path = path.with_suffix(extension)
-        if nmode is not None:
-            path_parts = str(path).split("_")
-            path_parts.insert(-1, nmode)
-            path = Path("_".join(path_parts))
+        path = href.strip("/") + "/" + _id + "." + extension.lstrip(".")
+        path_parts = str(path).split("_")
+        path_parts.insert(-1, nmode)
+        path = "_".join(path_parts)
         return str(path)
 
 
@@ -65,7 +61,8 @@ def __init__(self, h5_href: str) -> None:
         self.metadata = self.from_file(self.href)
 
     def from_file(self, href: str) -> Any:
-        return h5py.File(href, "r")
+        fs = fsspec.open(href)
+        return h5py.File(fs.open(), "r")
 
 
 class AnnotatedMetadata:
@@ -87,7 +84,7 @@ def parse_ann(self, ann_href: str) -> Dict[str, Any]:
             DEM Datum                                         (&)         = WGS-84
             becomes {"DEM_Datum": "WGS-84"}
         """
-        with open(ann_href, "r") as f:
+        with fsspec.open(ann_href, "r") as f:
             lines = f.readlines()
             data = {}
             for line in lines:

src/stactools/nisar_sim/stac.py

@@ -59,7 +59,7 @@ def create_collection() -> Collection:
     return collection
 
 
-def create_item(product_path: str, dither: str, nmode: str) -> Item:
+def create_item(product_href: str, dither: str, nmode: str) -> Item:
     """Create a STAC Item
 
     Args:
@@ -74,10 +74,10 @@ def create_item(product_path: str, dither: str, nmode: str) -> Item:
     Returns:
         Item: STAC Item object
     """
-    metalinks = MetadataLinks(product_path, dither, nmode)
+    metalinks = MetadataLinks(product_href, dither, nmode)
     h5_data = HDF5Metadata(metalinks.h5_href)
     ann_data = AnnotatedMetadata(metalinks.ann_a_href, metalinks.ann_b_href)
-    metadata = Metadata(product_path, metalinks.id, h5_data, ann_data)
+    metadata = Metadata(product_href, metalinks.id, h5_data, ann_data)
 
     item = Item(
         id=metalinks.id,

tests/conftest.py

@@ -0,0 +1,8 @@
+import pytest
+
+from stactools.nisar_sim.constants import NISAR_SIM_EXAMPLE_HREF
+
+
+@pytest.fixture
+def example_href() -> str:
+    return NISAR_SIM_EXAMPLE_HREF

tests/test_commands.py

@@ -39,12 +39,11 @@ def test_create_collection(self) -> None:
 
             collection.validate()
 
-    def test_create_item(self) -> None:
+    def test_create_item(self, example_href: str) -> None:
         with TemporaryDirectory() as tmp_dir:
             # Run your custom create-item command and validate
 
             # Example:
-            inpath = "tests/data-files/winnip_31604_12061_004_120717_L090_CX_07"
             destination = os.path.join(tmp_dir, "item.json")
             dither = "X"
             nmode = "129"
@@ -53,7 +52,7 @@ def test_create_item(self) -> None:
                 [
                     "nisarsim",
                     "create-item",
-                    f"{inpath}",
+                    example_href,
                     f"{destination}",
                     f"--dither {dither}",
                     f"--nmode {nmode}",

tests/test_metadata.py

@@ -0,0 +1,13 @@
+from stactools.nisar_sim import metadata
+
+
+def test_MetadataLinks() -> None:
+    href = (
+        "https://downloaduav.jpl.nasa.gov/Release2e/"
+        "Haywrd_14501_08037_007_080729_L090_CX_01/"
+    )
+    metalinks = metadata.MetadataLinks(href=href, dither="X", nmode="X")
+    assert (
+        metalinks.h5_href == "https://downloaduav.jpl.nasa.gov/Release2e/"
+        "Haywrd_14501_08037_007_080729_L090_CX_01/Haywrd_14501_08037_007_080729_L090_CX_X_01.h5"
+    )

tests/test_stac.py

@@ -15,11 +15,11 @@ def test_create_collection() -> None:
     collection.validate()
 
 
-def test_create_item() -> None:
+def test_create_item(example_href: str) -> None:
     # Write tests for each for the creation of STAC Items
     # Create the STAC Item...
     item = stac.create_item(
-        "tests/data-files/winnip_31604_12061_004_120717_L090_CX_07",
+        example_href,
         dither="X",
         nmode="129",
     )