Skip to content

Commit

Permalink
Address smoke test failure - use new method. (#16)
Browse files Browse the repository at this point in the history
  • Loading branch information
delucchi-cmu authored Jan 22, 2024
1 parent b58f0b9 commit 256558d
Showing 1 changed file with 4 additions and 3 deletions.
7 changes: 4 additions & 3 deletions src/lsdb_macauff/import_pipeline/map_reduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
from hipscat.io import file_io, paths
from hipscat.pixel_math.healpix_pixel import HealpixPixel
from hipscat.pixel_math.healpix_pixel_function import get_pixel_argsort
from hipscat_import.catalog.map_reduce import _get_pixel_directory, _iterate_input_file
from hipscat_import.catalog.map_reduce import _iterate_input_file
from hipscat_import.pipeline_resume_plan import get_pixel_cache_directory

from lsdb_macauff.import_pipeline.resume_plan import MacauffResumePlan

Expand Down Expand Up @@ -62,7 +63,7 @@ def split_associations(

filtered_data = data.filter(items=data_indexes, axis=0)

pixel_dir = _get_pixel_directory(tmp_path, pixel.order, pixel.pixel)
pixel_dir = get_pixel_cache_directory(tmp_path, pixel)
file_io.make_directory(pixel_dir, exist_ok=True)
output_file = file_io.append_paths_to_pointer(
pixel_dir, f"shard_{splitting_key}_{chunk_number}.parquet"
Expand All @@ -76,7 +77,7 @@ def split_associations(
def reduce_associations(left_pixel, tmp_path, catalog_path, reduce_key):
"""For all points determined to be in the target left_pixel, map them to the appropriate right_pixel
and aggregate into a single parquet file."""
inputs = _get_pixel_directory(tmp_path, left_pixel.order, left_pixel.pixel)
inputs = get_pixel_cache_directory(tmp_path, left_pixel)

if not file_io.directory_has_contents(inputs):
MacauffResumePlan.reducing_key_done(
Expand Down

0 comments on commit 256558d

Please sign in to comment.