Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

first working spec addition #4

Merged
merged 1 commit into from
Feb 23, 2024
Merged

first working spec addition #4

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 21 additions & 9 deletions src/tiled_ingestor/ingest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@
import logging
import sys

import h5py
from tiled.catalog.register import identity, register
from tiled.catalog import from_uri
import tiled.config
from tiled.adapters.hdf5 import HDF5Adapter, SWMR_DEFAULT
from tiled.structures.core import Spec
from tiled.utils import path_from_uri

logger = logging.getLogger(__name__)

Expand All @@ -15,6 +19,14 @@ def get_tiled_config(config_path: str):
return tiled.config.parse_configs(config_path)


def diamond_tomo_h5(data_uri, swmr=SWMR_DEFAULT, libver="latest", specs=None, **kwargs):
specs = specs or []
specs.append(Spec("a_spec", "a_version"))
filepath = path_from_uri(data_uri)
file = h5py.File(filepath, "r", swmr=swmr, libver=libver)
return HDF5Adapter.from_file(file, specs=specs, **kwargs)


async def process_file(
file_path: str,
tiled_config: dict,
Expand Down Expand Up @@ -57,11 +69,11 @@ async def process_file(
matching_tree["tree"] == "catalog"
), f"Matching tiled tree {tiled_config_tree_path} is not a catalog"

# using thre tree in the configuration, generate a catalog(adapter)
# using the tree in the configuration, generate a catalog(adapter)
catalog_adapter = from_uri(
matching_tree["args"]["uri"],
readable_storage=matching_tree["args"]["readable_storage"],
adapters_by_mimetype=matching_tree["args"].get("adapters_by_mimetype"),

)

# Register with tiled. This writes entries into the database for all of the nodes down to the data node
Expand All @@ -71,6 +83,7 @@ async def process_file(
path=file_path,
prefix=path_prefix,
overwrite=False,
adapters_by_mimetype=matching_tree["args"].get("adapters_by_mimetype")
)


Expand All @@ -84,23 +97,22 @@ async def process_file(
tiled_config = get_tiled_config("../mlex_tomo_framework/tiled/deploy/config")
asyncio.run(
process_file(
"../mlex_tomo_framework/data/tiled_storage/beamlines/8.3.2/recons/rec20240207_120829_test_no_xrays_n1313",
"../mlex_tomo_framework/data/tiled_storage/recons/nexus-example.nxs",
tiled_config,
path_prefix="/beamlines/8.3.2/recons/",
path_prefix="/recons",
# specs=[{"name": "ANexus", "version": "sdfsdf"}]
)
)
else:
from pprint import pprint
import os

pprint(os.environ)
tiled_config = get_tiled_config(
"/tiled_storage/beamlines/8.3.2/recons/rec20240207_120829_test_no_xrays_n1313"
)
tiled_config = get_tiled_config("../mlex_tomo_framework/tiled/deploy/config")
asyncio.run(
process_file(
# "/tiled_storage/beamlines/8.3.2/recons/rec20240207_120550_test_no_xrays_n257",
"/tiled_storage/recons/rec20240207_120550_test_no_xrays_n257",
tiled_config,
path_prefix="/beamlines/8.3.2/recons/",
path_prefix="/recons",
)
)
Loading