Skip to content

Commit

Permalink
Support to query params on http filesystem
Browse files Browse the repository at this point in the history
  • Loading branch information
Schwarzam committed Jun 13, 2024
1 parent 73948db commit 7ec0afa
Show file tree
Hide file tree
Showing 3 changed files with 27 additions and 2 deletions.
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -157,4 +157,7 @@ _html/
.initialize_new_project.sh

# large, unused fits files
point_map.fits
point_map.fits

# test notebook
dev/test.ipynb
5 changes: 4 additions & 1 deletion src/lsdb/loaders/hipscat/abstract_catalog_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,10 @@ def _get_paths_from_pixels(
self, catalog: HCHealpixDataset, ordered_pixels: List[HealpixPixel]
) -> List[hc.io.FilePointer]:
paths = hc.io.paths.pixel_catalog_files(
catalog.catalog_base_dir, ordered_pixels, storage_options=self.storage_options
catalog.catalog_base_dir,
ordered_pixels,
self.config.make_query_url_params(),
self.storage_options,
)
return paths

Expand Down
19 changes: 19 additions & 0 deletions src/lsdb/loaders/hipscat/hipscat_loading_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,22 @@ def get_dtype_mapper(self) -> Callable | None:
elif self.dtype_backend == "numpy_nullable":
mapper = _arrow_dtype_mapping().get
return mapper

def make_query_url_params(self) -> str:
"""
Generates a dictionary of URL parameters with `columns` and `filters` attributes.
"""
url_params = {}

if self.columns and len(self.columns) > 0:
url_params["columns"] = self.columns

if "filters" in self.kwargs:
url_params["filters"] = []
for filter in self.kwargs["filters"]:
# This is how hipscat expects the filters to add to the url
url_params["filters"].append(f"{filter[0]}{filter[1]}{filter[2]}")

# The filter should be only computed on the server side
self.kwargs.pop("filters", None)
return url_params

0 comments on commit 7ec0afa

Please sign in to comment.