From 9038724bb0f688cc44b4cc23e154e7f6d17d47c5 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi <113376043+delucchi-cmu@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:13:02 -0400 Subject: [PATCH] Hats renaming (#408) * Initial rename to HATS (#390) * Initial renaming * Couple more places. * "hats-sharded" * Remove text about healpix_29 uniqueness. * Create HiPS-style properties file (#391) * Initial work toward properties file. * Apply suggestions from code review Co-authored-by: Sandro Campos * Code review comments. --------- Co-authored-by: Sandro Campos * Initial pipeline for converting hipscat -> hats (#392) * Initial work toward properties file. * Initial pipeline for converting hipscat -> hats * Apply suggestions from code review Co-authored-by: Derek T. Jones --------- Co-authored-by: Derek T. Jones * Follow-up on run_conversion.py The `:=` operator obviates the line following, but I wasn't able to make that part of my PR suggestion. * Set new and additional properties. (#396) * Initial work toward properties file. * Initial pipeline for converting hipscat -> hats * Remove provenance, remove methods, use new constants * Set new and additional properties. * Clean up merge * Improve readability of literals. * Set additional properties on conversion (#397) * use spatial index from hats * Update size estimate on converted catalog. (#402) * Update size estimate on converted catalog. * Style cleanup and enforcement. * Whomp whomp * Reduce code duplication * Just don't bother with test files for mypy * Insert dataset dir, and use general ra/dec columns (#406) * Insert dataset dir, and use general ra/dec columns * Update src/hats_import/hipscat_conversion/run_conversion.py Co-authored-by: Sandro Campos * Update deps for renamed repo --------- Co-authored-by: Sandro Campos * Apply suggestions from code review Co-authored-by: Sandro Campos * Remove branch from workflows --------- Co-authored-by: Sandro Campos Co-authored-by: Derek T. Jones Co-authored-by: Sean McGuire Co-authored-by: Sean McGuire <123987820+smcguire-cmu@users.noreply.github.com> --- .copier-answers.yml | 4 +- .github/pull_request_template.md | 2 +- .github/workflows/publish-to-pypi.yml | 2 +- .github/workflows/testing-and-coverage.yml | 2 +- .pre-commit-config.yaml | 2 +- README.md | 24 +-- benchmarks/asv.conf.json | 6 +- benchmarks/benchmarks.py | 4 +- docs/catalogs/arguments.rst | 43 ++--- docs/catalogs/public/allwise.rst | 6 +- docs/catalogs/public/neowise.rst | 6 +- docs/catalogs/public/panstarrs.rst | 6 +- docs/catalogs/public/sdss.rst | 4 +- docs/catalogs/public/tic.rst | 6 +- docs/catalogs/public/zubercal.rst | 6 +- docs/catalogs/temp_files.rst | 12 +- docs/conf.py | 8 +- docs/guide/contact.rst | 4 +- docs/guide/contributing.rst | 2 +- docs/guide/dask_on_ray.rst | 4 +- docs/guide/index_table.rst | 35 ++-- docs/guide/margin_cache.rst | 16 +- docs/index.rst | 16 +- docs/notebooks/estimate_pixel_threshold.ipynb | 14 +- docs/notebooks/unequal_schema.ipynb | 12 +- docs/requirements.txt | 2 +- pyproject.toml | 17 +- requirements.txt | 2 +- src/.pylintrc | 2 + .../__init__.py | 2 +- .../catalog/__init__.py | 0 .../catalog/arguments.py | 79 +++----- .../catalog/file_readers.py | 17 +- .../catalog/map_reduce.py | 87 +++++---- .../catalog/resume_plan.py | 24 +-- .../catalog/run_import.py | 43 ++--- .../catalog/sparse_histogram.py | 2 +- .../hipscat_conversion/__init__.py | 4 + .../hipscat_conversion/arguments.py | 28 +++ .../hipscat_conversion/run_conversion.py | 177 ++++++++++++++++++ .../index/__init__.py | 2 +- .../index/arguments.py | 36 ++-- .../index/map_reduce.py | 16 +- .../index/run_index.py | 22 +-- .../margin_cache/__init__.py | 0 .../margin_cache/margin_cache.py | 24 +-- .../margin_cache/margin_cache_arguments.py | 42 ++--- .../margin_cache/margin_cache_map_reduce.py | 37 ++-- .../margin_cache/margin_cache_resume_plan.py | 20 +- .../pipeline.py | 26 +-- .../pipeline_resume_plan.py | 6 +- src/{hipscat_import => hats_import}/py.typed | 0 .../runtime_arguments.py | 79 +++++--- .../soap/__init__.py | 0 .../soap/arguments.py | 45 ++--- .../soap/map_reduce.py | 18 +- .../soap/resume_plan.py | 28 +-- .../soap/run_soap.py | 21 +-- .../verification/__init__.py | 0 .../verification/arguments.py | 15 +- .../verification/run_verification.py | 4 +- tests/conftest.py | 2 +- .../{hipscat_import => }/data/blank/blank.csv | 0 tests/data/generate_data.ipynb | 145 ++++++++++++++ .../Norder=0/Dir=0/Npix=11.parquet | Bin .../small_sky_object_catalog/_common_metadata | Bin .../small_sky_object_catalog/_metadata | Bin .../catalog_info.json | 0 .../partition_info.csv | 0 .../provenance_info.json | 0 .../Norder=0/Dir=0/Npix=4.parquet | Bin .../Norder=1/Dir=0/Npix=47.parquet | Bin .../Norder=2/Dir=0/Npix=176.parquet | Bin .../Norder=2/Dir=0/Npix=177.parquet | Bin .../Norder=2/Dir=0/Npix=178.parquet | Bin .../Norder=2/Dir=0/Npix=179.parquet | Bin .../Norder=2/Dir=0/Npix=180.parquet | Bin .../Norder=2/Dir=0/Npix=181.parquet | Bin .../Norder=2/Dir=0/Npix=182.parquet | Bin .../Norder=2/Dir=0/Npix=183.parquet | Bin .../Norder=2/Dir=0/Npix=184.parquet | Bin .../Norder=2/Dir=0/Npix=185.parquet | Bin .../Norder=2/Dir=0/Npix=186.parquet | Bin .../Norder=2/Dir=0/Npix=187.parquet | Bin .../small_sky_source_catalog/_common_metadata | Bin .../small_sky_source_catalog/_metadata | Bin .../catalog_info.json | 0 .../partition_info.csv | 0 .../small_sky_source_catalog/point_map.fits | Bin .../provenance_info.json | 0 .../indexed_files/csv_list_double_1_of_2.txt | 3 + .../indexed_files/csv_list_double_2_of_2.txt | 3 + tests/data/indexed_files/csv_list_single.txt | 6 + .../indexed_files/parquet_list_single.txt | 5 + .../data/margin_pairs/negative_pairs.csv | 0 .../margin_pairs/small_sky_source_pairs.csv | 0 .../data/mixed_schema/input_01.csv | 0 .../data/mixed_schema/input_02.csv | 0 .../data/mixed_schema/schema.parquet | Bin .../order_0/dir_0/pixel_11/shard_0_0.parquet | Bin .../order_0/dir_0/pixel_11/shard_1_0.parquet | Bin .../order_0/dir_0/pixel_11/shard_2_0.parquet | Bin .../order_0/dir_0/pixel_11/shard_3_0.parquet | Bin .../order_0/dir_0/pixel_11/shard_4_0.parquet | Bin .../order_1/dir_0/pixel_44/shard_0_0.parquet | Bin .../order_1/dir_0/pixel_44/shard_1_0.parquet | Bin .../order_1/dir_0/pixel_44/shard_2_0.parquet | Bin .../order_1/dir_0/pixel_44/shard_3_0.parquet | Bin .../order_1/dir_0/pixel_44/shard_4_0.parquet | Bin .../order_1/dir_0/pixel_45/shard_0_0.parquet | Bin .../order_1/dir_0/pixel_45/shard_1_0.parquet | Bin .../order_1/dir_0/pixel_45/shard_2_0.parquet | Bin .../order_1/dir_0/pixel_45/shard_3_0.parquet | Bin .../order_1/dir_0/pixel_45/shard_4_0.parquet | Bin .../order_1/dir_0/pixel_46/shard_0_0.parquet | Bin .../order_1/dir_0/pixel_46/shard_1_0.parquet | Bin .../order_1/dir_0/pixel_46/shard_2_0.parquet | Bin .../order_1/dir_0/pixel_46/shard_3_0.parquet | Bin .../order_1/dir_0/pixel_46/shard_4_0.parquet | Bin .../order_1/dir_0/pixel_47/shard_0_0.parquet | Bin .../order_1/dir_0/pixel_47/shard_1_0.parquet | Bin .../order_1/dir_0/pixel_47/shard_2_0.parquet | Bin .../order_1/dir_0/pixel_47/shard_3_0.parquet | Bin .../order_1/dir_0/pixel_47/shard_4_0.parquet | Bin .../resume/Norder=0/Dir=0/Npix=11.parquet | Bin .../resume/Norder=1/Dir=0/Npix=44.parquet | Bin .../resume/Norder=1/Dir=0/Npix=45.parquet | Bin .../resume/Norder=1/Dir=0/Npix=46.parquet | Bin .../resume/Norder=1/Dir=0/Npix=47.parquet | Bin .../intermediate/mapping_histogram.binary | Bin .../data/small_sky/catalog.csv | 0 .../dataset/Norder=0/Dir=0/Npix=11.parquet | Bin 0 -> 8913 bytes .../dataset/_common_metadata | Bin 0 -> 3983 bytes .../dataset/_metadata | Bin 0 -> 5175 bytes .../partition_info.csv | 2 + .../small_sky_object_catalog/point_map.fits | Bin 0 -> 57600 bytes .../data/small_sky_object_catalog/properties | 14 ++ .../data/small_sky_parts/catalog_00_of_05.csv | 0 .../data/small_sky_parts/catalog_01_of_05.csv | 0 .../data/small_sky_parts/catalog_02_of_05.csv | 0 .../data/small_sky_parts/catalog_03_of_05.csv | 0 .../data/small_sky_parts/catalog_04_of_05.csv | 0 .../data/small_sky_parts/catalog_10_of_05.csv | 0 .../small_sky_source/small_sky_source.csv | 0 .../dataset/Norder=0/Dir=0/Npix=4.parquet | Bin 0 -> 11523 bytes .../dataset/Norder=1/Dir=0/Npix=47.parquet | Bin 0 -> 137676 bytes .../dataset/Norder=2/Dir=0/Npix=176.parquet | Bin 0 -> 29117 bytes .../dataset/Norder=2/Dir=0/Npix=177.parquet | Bin 0 -> 89086 bytes .../dataset/Norder=2/Dir=0/Npix=178.parquet | Bin 0 -> 95714 bytes .../dataset/Norder=2/Dir=0/Npix=179.parquet | Bin 0 -> 103119 bytes .../dataset/Norder=2/Dir=0/Npix=180.parquet | Bin 0 -> 43359 bytes .../dataset/Norder=2/Dir=0/Npix=181.parquet | Bin 0 -> 56287 bytes .../dataset/Norder=2/Dir=0/Npix=182.parquet | Bin 0 -> 75097 bytes .../dataset/Norder=2/Dir=0/Npix=183.parquet | Bin 0 -> 69687 bytes .../dataset/Norder=2/Dir=0/Npix=184.parquet | Bin 0 -> 82739 bytes .../dataset/Norder=2/Dir=0/Npix=185.parquet | Bin 0 -> 167156 bytes .../dataset/Norder=2/Dir=0/Npix=186.parquet | Bin 0 -> 32523 bytes .../dataset/Norder=2/Dir=0/Npix=187.parquet | Bin 0 -> 44887 bytes .../dataset/_common_metadata | Bin 0 -> 5490 bytes .../dataset/_metadata | Bin 0 -> 30446 bytes .../partition_info.csv | 15 ++ .../small_sky_source_catalog/point_map.fits | Bin 0 -> 57600 bytes .../data/small_sky_source_catalog/properties | 14 ++ .../data/soap_intermediate/0_4.csv | 0 .../data/soap_intermediate/1_47.csv | 0 .../data/soap_intermediate/2_176.csv | 0 .../data/soap_intermediate/2_177.csv | 0 .../data/soap_intermediate/2_178.csv | 0 .../data/soap_intermediate/2_179.csv | 0 .../data/soap_intermediate/2_180.csv | 0 .../data/soap_intermediate/2_181.csv | 0 .../data/soap_intermediate/2_182.csv | 0 .../data/soap_intermediate/2_183.csv | 0 .../data/soap_intermediate/2_184.csv | 0 .../data/soap_intermediate/2_185.csv | 0 .../data/soap_intermediate/2_186.csv | 0 .../data/soap_intermediate/2_187.csv | 0 .../order_0/dir_0/pixel_11/source_0_4.parquet | Bin .../dir_0/pixel_11/source_1_47.parquet | Bin .../dir_0/pixel_11/source_2_176.parquet | Bin .../dir_0/pixel_11/source_2_177.parquet | Bin .../dir_0/pixel_11/source_2_178.parquet | Bin .../dir_0/pixel_11/source_2_179.parquet | Bin .../dir_0/pixel_11/source_2_180.parquet | Bin .../dir_0/pixel_11/source_2_181.parquet | Bin .../dir_0/pixel_11/source_2_182.parquet | Bin .../dir_0/pixel_11/source_2_183.parquet | Bin .../dir_0/pixel_11/source_2_184.parquet | Bin .../dir_0/pixel_11/source_2_185.parquet | Bin .../dir_0/pixel_11/source_2_186.parquet | Bin .../dir_0/pixel_11/source_2_187.parquet | Bin .../data/test_formats/catalog.csv.gz | Bin .../data/test_formats/catalog.starr | 0 .../data/test_formats/catalog.zip | Bin .../data/test_formats/gaia_epoch.ecsv | 0 .../data/test_formats/gaia_minimum.csv | 0 .../test_formats/gaia_minimum_schema.parquet | Bin .../data/test_formats/headers.csv | 0 .../test_formats/healpix_29_index.parquet | Bin 0 -> 4027 bytes .../data/test_formats/macauff_metadata.yaml | 0 .../data/test_formats/pandasindex.parquet | Bin .../data/test_formats/pipe_delimited.csv | 0 .../data/test_formats/small_sky.fits | Bin tests/data/test_formats/spatial_index.csv | 132 +++++++++++++ .../catalog/test_argument_validation.py | 67 ++----- .../catalog/test_file_readers.py | 52 +---- .../catalog/test_map_reduce.py | 44 ++--- .../catalog/test_resume_plan.py | 4 +- .../catalog/test_run_import.py | 43 +++-- .../catalog/test_run_round_trip.py | 106 +++++------ .../catalog/test_sparse_histogram.py | 2 +- .../conftest.py | 16 +- .../hipscat_conversion/test_run_conversion.py | 124 ++++++++++++ .../index/test_index_argument.py | 35 +--- .../index/test_index_map_reduce.py | 36 ++-- .../index/test_run_index.py | 42 +++-- .../test_arguments_margin_cache.py | 29 +-- .../margin_cache/test_margin_cache.py | 27 ++- .../test_margin_cache_map_reduce.py | 74 ++++++-- .../test_margin_cache_resume_plan.py | 10 +- .../margin_cache/test_margin_round_trip.py | 26 +-- .../soap/conftest.py | 6 +- .../soap/test_run_soap.py | 62 ++---- .../soap/test_soap_arguments.py | 2 +- .../soap/test_soap_map_reduce.py | 10 +- .../soap/test_soap_resume_plan.py | 19 +- tests/hats_import/test_packaging.py | 6 + .../test_pipeline_resume_plan.py | 2 +- .../test_runtime_arguments.py | 53 +++++- .../verification/test_run_verification.py | 4 +- .../test_verification_arguments.py | 27 +-- .../indexed_files/csv_list_double_1_of_2.txt | 3 - .../indexed_files/csv_list_double_2_of_2.txt | 3 - .../data/indexed_files/csv_list_single.txt | 6 - .../indexed_files/parquet_list_single.txt | 5 - .../small_sky_object_catalog/point_map.fits | Bin 1581120 -> 0 bytes .../data/test_formats/hipscat_index.csv | 132 ------------- .../data/test_formats/hipscat_index.parquet | Bin 4034 -> 0 bytes tests/hipscat_import/test_packaging.py | 6 - 239 files changed, 1446 insertions(+), 1151 deletions(-) rename src/{hipscat_import => hats_import}/__init__.py (64%) rename src/{hipscat_import => hats_import}/catalog/__init__.py (100%) rename src/{hipscat_import => hats_import}/catalog/arguments.py (68%) rename src/{hipscat_import => hats_import}/catalog/file_readers.py (95%) rename src/{hipscat_import => hats_import}/catalog/map_reduce.py (82%) rename src/{hipscat_import => hats_import}/catalog/resume_plan.py (95%) rename src/{hipscat_import => hats_import}/catalog/run_import.py (81%) rename src/{hipscat_import => hats_import}/catalog/sparse_histogram.py (98%) create mode 100644 src/hats_import/hipscat_conversion/__init__.py create mode 100644 src/hats_import/hipscat_conversion/arguments.py create mode 100644 src/hats_import/hipscat_conversion/run_conversion.py rename src/{hipscat_import => hats_import}/index/__init__.py (54%) rename src/{hipscat_import => hats_import}/index/arguments.py (73%) rename src/{hipscat_import => hats_import}/index/map_reduce.py (82%) rename src/{hipscat_import => hats_import}/index/run_index.py (52%) rename src/{hipscat_import => hats_import}/margin_cache/__init__.py (100%) rename src/{hipscat_import => hats_import}/margin_cache/margin_cache.py (80%) rename src/{hipscat_import => hats_import}/margin_cache/margin_cache_arguments.py (69%) rename src/{hipscat_import => hats_import}/margin_cache/margin_cache_map_reduce.py (81%) rename src/{hipscat_import => hats_import}/margin_cache/margin_cache_resume_plan.py (90%) rename src/{hipscat_import => hats_import}/pipeline.py (75%) rename src/{hipscat_import => hats_import}/pipeline_resume_plan.py (99%) rename src/{hipscat_import => hats_import}/py.typed (100%) rename src/{hipscat_import => hats_import}/runtime_arguments.py (73%) rename src/{hipscat_import => hats_import}/soap/__init__.py (100%) rename src/{hipscat_import => hats_import}/soap/arguments.py (54%) rename src/{hipscat_import => hats_import}/soap/map_reduce.py (93%) rename src/{hipscat_import => hats_import}/soap/resume_plan.py (90%) rename src/{hipscat_import => hats_import}/soap/run_soap.py (76%) rename src/{hipscat_import => hats_import}/verification/__init__.py (100%) rename src/{hipscat_import => hats_import}/verification/arguments.py (74%) rename src/{hipscat_import => hats_import}/verification/run_verification.py (82%) rename tests/{hipscat_import => }/data/blank/blank.csv (100%) create mode 100644 tests/data/generate_data.ipynb rename tests/{hipscat_import/data => data/hipscat}/small_sky_object_catalog/Norder=0/Dir=0/Npix=11.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_object_catalog/_common_metadata (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_object_catalog/_metadata (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_object_catalog/catalog_info.json (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_object_catalog/partition_info.csv (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_object_catalog/provenance_info.json (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=0/Dir=0/Npix=4.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=1/Dir=0/Npix=47.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=176.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=177.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=178.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=179.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=180.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=181.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=182.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=183.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=184.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=185.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=186.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/Norder=2/Dir=0/Npix=187.parquet (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/_common_metadata (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/_metadata (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/catalog_info.json (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/partition_info.csv (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/point_map.fits (100%) rename tests/{hipscat_import/data => data/hipscat}/small_sky_source_catalog/provenance_info.json (100%) create mode 100644 tests/data/indexed_files/csv_list_double_1_of_2.txt create mode 100644 tests/data/indexed_files/csv_list_double_2_of_2.txt create mode 100644 tests/data/indexed_files/csv_list_single.txt create mode 100644 tests/data/indexed_files/parquet_list_single.txt rename tests/{hipscat_import => }/data/margin_pairs/negative_pairs.csv (100%) rename tests/{hipscat_import => }/data/margin_pairs/small_sky_source_pairs.csv (100%) rename tests/{hipscat_import => }/data/mixed_schema/input_01.csv (100%) rename tests/{hipscat_import => }/data/mixed_schema/input_02.csv (100%) rename tests/{hipscat_import => }/data/mixed_schema/schema.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_0/dir_0/pixel_11/shard_0_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_0/dir_0/pixel_11/shard_1_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_0/dir_0/pixel_11/shard_2_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_0/dir_0/pixel_11/shard_3_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_0/dir_0/pixel_11/shard_4_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_44/shard_0_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_44/shard_1_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_44/shard_2_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_44/shard_3_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_44/shard_4_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_45/shard_0_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_45/shard_1_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_45/shard_2_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_45/shard_3_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_45/shard_4_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_46/shard_0_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_46/shard_1_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_46/shard_2_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_46/shard_3_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_46/shard_4_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_47/shard_0_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_47/shard_1_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_47/shard_2_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_47/shard_3_0.parquet (100%) rename tests/{hipscat_import => }/data/parquet_shards/order_1/dir_0/pixel_47/shard_4_0.parquet (100%) rename tests/{hipscat_import => }/data/resume/Norder=0/Dir=0/Npix=11.parquet (100%) rename tests/{hipscat_import => }/data/resume/Norder=1/Dir=0/Npix=44.parquet (100%) rename tests/{hipscat_import => }/data/resume/Norder=1/Dir=0/Npix=45.parquet (100%) rename tests/{hipscat_import => }/data/resume/Norder=1/Dir=0/Npix=46.parquet (100%) rename tests/{hipscat_import => }/data/resume/Norder=1/Dir=0/Npix=47.parquet (100%) rename tests/{hipscat_import => }/data/resume/intermediate/mapping_histogram.binary (100%) rename tests/{hipscat_import => }/data/small_sky/catalog.csv (100%) create mode 100644 tests/data/small_sky_object_catalog/dataset/Norder=0/Dir=0/Npix=11.parquet create mode 100644 tests/data/small_sky_object_catalog/dataset/_common_metadata create mode 100644 tests/data/small_sky_object_catalog/dataset/_metadata create mode 100644 tests/data/small_sky_object_catalog/partition_info.csv create mode 100644 tests/data/small_sky_object_catalog/point_map.fits create mode 100644 tests/data/small_sky_object_catalog/properties rename tests/{hipscat_import => }/data/small_sky_parts/catalog_00_of_05.csv (100%) rename tests/{hipscat_import => }/data/small_sky_parts/catalog_01_of_05.csv (100%) rename tests/{hipscat_import => }/data/small_sky_parts/catalog_02_of_05.csv (100%) rename tests/{hipscat_import => }/data/small_sky_parts/catalog_03_of_05.csv (100%) rename tests/{hipscat_import => }/data/small_sky_parts/catalog_04_of_05.csv (100%) rename tests/{hipscat_import => }/data/small_sky_parts/catalog_10_of_05.csv (100%) rename tests/{hipscat_import => }/data/small_sky_source/small_sky_source.csv (100%) create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=0/Dir=0/Npix=4.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=1/Dir=0/Npix=47.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=176.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=177.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=178.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=179.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=180.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=181.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=182.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=183.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=184.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=185.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=186.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/Norder=2/Dir=0/Npix=187.parquet create mode 100644 tests/data/small_sky_source_catalog/dataset/_common_metadata create mode 100644 tests/data/small_sky_source_catalog/dataset/_metadata create mode 100644 tests/data/small_sky_source_catalog/partition_info.csv create mode 100644 tests/data/small_sky_source_catalog/point_map.fits create mode 100644 tests/data/small_sky_source_catalog/properties rename tests/{hipscat_import => }/data/soap_intermediate/0_4.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/1_47.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_176.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_177.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_178.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_179.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_180.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_181.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_182.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_183.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_184.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_185.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_186.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/2_187.csv (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_0_4.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_1_47.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_176.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_177.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_178.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_179.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_180.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_181.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_182.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_183.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_184.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_185.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_186.parquet (100%) rename tests/{hipscat_import => }/data/soap_intermediate/order_0/dir_0/pixel_11/source_2_187.parquet (100%) rename tests/{hipscat_import => }/data/test_formats/catalog.csv.gz (100%) rename tests/{hipscat_import => }/data/test_formats/catalog.starr (100%) rename tests/{hipscat_import => }/data/test_formats/catalog.zip (100%) rename tests/{hipscat_import => }/data/test_formats/gaia_epoch.ecsv (100%) rename tests/{hipscat_import => }/data/test_formats/gaia_minimum.csv (100%) rename tests/{hipscat_import => }/data/test_formats/gaia_minimum_schema.parquet (100%) rename tests/{hipscat_import => }/data/test_formats/headers.csv (100%) create mode 100644 tests/data/test_formats/healpix_29_index.parquet rename tests/{hipscat_import => }/data/test_formats/macauff_metadata.yaml (100%) rename tests/{hipscat_import => }/data/test_formats/pandasindex.parquet (100%) rename tests/{hipscat_import => }/data/test_formats/pipe_delimited.csv (100%) rename tests/{hipscat_import => }/data/test_formats/small_sky.fits (100%) create mode 100644 tests/data/test_formats/spatial_index.csv rename tests/{hipscat_import => hats_import}/catalog/test_argument_validation.py (79%) rename tests/{hipscat_import => hats_import}/catalog/test_file_readers.py (84%) rename tests/{hipscat_import => hats_import}/catalog/test_map_reduce.py (93%) rename tests/{hipscat_import => hats_import}/catalog/test_resume_plan.py (98%) rename tests/{hipscat_import => hats_import}/catalog/test_run_import.py (89%) rename tests/{hipscat_import => hats_import}/catalog/test_run_round_trip.py (87%) rename tests/{hipscat_import => hats_import}/catalog/test_sparse_histogram.py (97%) rename tests/{hipscat_import => hats_import}/conftest.py (95%) create mode 100644 tests/hats_import/hipscat_conversion/test_run_conversion.py rename tests/{hipscat_import => hats_import}/index/test_index_argument.py (84%) rename tests/{hipscat_import => hats_import}/index/test_index_map_reduce.py (82%) rename tests/{hipscat_import => hats_import}/index/test_run_index.py (73%) rename tests/{hipscat_import => hats_import}/margin_cache/test_arguments_margin_cache.py (80%) rename tests/{hipscat_import => hats_import}/margin_cache/test_margin_cache.py (70%) rename tests/{hipscat_import => hats_import}/margin_cache/test_margin_cache_map_reduce.py (79%) rename tests/{hipscat_import => hats_import}/margin_cache/test_margin_cache_resume_plan.py (93%) rename tests/{hipscat_import => hats_import}/margin_cache/test_margin_round_trip.py (81%) rename tests/{hipscat_import => hats_import}/soap/conftest.py (92%) rename tests/{hipscat_import => hats_import}/soap/test_run_soap.py (71%) rename tests/{hipscat_import => hats_import}/soap/test_soap_arguments.py (98%) rename tests/{hipscat_import => hats_import}/soap/test_soap_map_reduce.py (95%) rename tests/{hipscat_import => hats_import}/soap/test_soap_resume_plan.py (90%) create mode 100644 tests/hats_import/test_packaging.py rename tests/{hipscat_import => hats_import}/test_pipeline_resume_plan.py (98%) rename tests/{hipscat_import => hats_import}/test_runtime_arguments.py (70%) rename tests/{hipscat_import => hats_import}/verification/test_run_verification.py (85%) rename tests/{hipscat_import => hats_import}/verification/test_verification_arguments.py (76%) delete mode 100644 tests/hipscat_import/data/indexed_files/csv_list_double_1_of_2.txt delete mode 100644 tests/hipscat_import/data/indexed_files/csv_list_double_2_of_2.txt delete mode 100644 tests/hipscat_import/data/indexed_files/csv_list_single.txt delete mode 100644 tests/hipscat_import/data/indexed_files/parquet_list_single.txt delete mode 100644 tests/hipscat_import/data/small_sky_object_catalog/point_map.fits delete mode 100644 tests/hipscat_import/data/test_formats/hipscat_index.csv delete mode 100644 tests/hipscat_import/data/test_formats/hipscat_index.parquet delete mode 100644 tests/hipscat_import/test_packaging.py diff --git a/.copier-answers.yml b/.copier-answers.yml index 44cede77..886b28f1 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -15,9 +15,9 @@ include_benchmarks: true include_docs: true include_notebooks: true mypy_type_checking: basic -package_name: hipscat_import +package_name: hats_import project_license: BSD -project_name: hipscat-import +project_name: hats-import project_organization: astronomy-commons python_versions: - '3.9' diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6a7b51dd..8d01c723 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -26,7 +26,7 @@ If it fixes an open issue, please link to the issue here. If this PR closes an i ## Code Quality -- [ ] I have read the [Contribution Guide](https://hipscat-import.readthedocs.io/en/stable/guide/contributing.html) and [LINCC Frameworks Code of Conduct](https://lsstdiscoveryalliance.org/programs/lincc-frameworks/code-conduct/) +- [ ] I have read the [Contribution Guide](https://hats-import.readthedocs.io/en/stable/guide/contributing.html) and [LINCC Frameworks Code of Conduct](https://lsstdiscoveryalliance.org/programs/lincc-frameworks/code-conduct/) - [ ] My code follows the code style of this project - [ ] My code builds (or compiles) cleanly without any errors or warnings - [ ] My code contains relevant comments and necessary documentation diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 49231cf6..ca15234a 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -32,7 +32,7 @@ jobs: python -m pip install --upgrade pip pip install . - name: Create lock requirements file - run: pip list --format=freeze --exclude "hipscat-import" > requirements.txt + run: pip list --format=freeze --exclude "hats-import" > requirements.txt - name: Install dev dependencies run: pip install .[dev] - name: Run unit tests with pytest diff --git a/.github/workflows/testing-and-coverage.yml b/.github/workflows/testing-and-coverage.yml index bb6c1668..6a2a7c7a 100644 --- a/.github/workflows/testing-and-coverage.yml +++ b/.github/workflows/testing-and-coverage.yml @@ -31,7 +31,7 @@ jobs: if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Run unit tests with pytest run: | - python -m pytest tests --cov=hipscat_import --cov-report=xml + python -m pytest tests --cov=hats_import --cov-report=xml - name: Run dask-on-ray tests with pytest run: | python -m pytest tests --use_ray diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d06c1c1a..29bd788c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -101,7 +101,7 @@ repos: entry: mypy language: system types: [python] - files: ^(src|tests)/ + files: ^src/ args: [ "--ignore-missing-imports", # Ignore imports without type hints diff --git a/README.md b/README.md index a7f2805b..b4d56657 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,33 @@ -# hipscat-import +# hats-import [![Template](https://img.shields.io/badge/Template-LINCC%20Frameworks%20Python%20Project%20Template-brightgreen)](https://lincc-ppt.readthedocs.io/en/stable/) -[![PyPI](https://img.shields.io/pypi/v/hipscat-import?color=blue&logo=pypi&logoColor=white)](https://pypi.org/project/hipscat-import/) -[![Conda](https://img.shields.io/conda/vn/conda-forge/hipscat-import.svg?color=blue&logo=condaforge&logoColor=white)](https://anaconda.org/conda-forge/hipscat-import) +[![PyPI](https://img.shields.io/pypi/v/hats-import?color=blue&logo=pypi&logoColor=white)](https://pypi.org/project/hats-import/) +[![Conda](https://img.shields.io/conda/vn/conda-forge/hats-import.svg?color=blue&logo=condaforge&logoColor=white)](https://anaconda.org/conda-forge/hats-import) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/astronomy-commons/hipscat-import/smoke-test.yml)](https://github.com/astronomy-commons/hipscat-import/actions/workflows/smoke-test.yml) -[![codecov](https://codecov.io/gh/astronomy-commons/hipscat-import/branch/main/graph/badge.svg)](https://codecov.io/gh/astronomy-commons/hipscat-import) -[![Read the Docs](https://img.shields.io/readthedocs/hipscat-import)](https://hipscat-import.readthedocs.io/) +[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/astronomy-commons/hats-import/smoke-test.yml)](https://github.com/astronomy-commons/hats-import/actions/workflows/smoke-test.yml) +[![codecov](https://codecov.io/gh/astronomy-commons/hats-import/branch/main/graph/badge.svg)](https://codecov.io/gh/astronomy-commons/hats-import) +[![Read the Docs](https://img.shields.io/readthedocs/hats-import)](https://hats-import.readthedocs.io/) -## HiPSCat import - Utility for ingesting large survey data into HiPSCat structure. +## HATS import - Utility for ingesting large survey data into HATS structure. -Check out our [ReadTheDocs site](https://hipscat-import.readthedocs.io/en/stable/) +Check out our [ReadTheDocs site](https://hats-import.readthedocs.io/en/stable/) for more information on partitioning, installation, and contributing. See related projects: -* HiPSCat ([on GitHub](https://github.com/astronomy-commons/hipscat)) - ([on ReadTheDocs](https://hipscat.readthedocs.io/en/stable/)) +* HATS ([on GitHub](https://github.com/astronomy-commons/hats)) + ([on ReadTheDocs](https://hats.readthedocs.io/en/stable/)) * LSDB ([on GitHub](https://github.com/astronomy-commons/lsdb)) ([on ReadTheDocs](https://lsdb.readthedocs.io/en/stable/)) ## Contributing -[![GitHub issue custom search in repo](https://img.shields.io/github/issues-search/astronomy-commons/hipscat-import?color=purple&label=Good%20first%20issues&query=is%3Aopen%20label%3A%22good%20first%20issue%22)](https://github.com/astronomy-commons/hipscat-import/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) +[![GitHub issue custom search in repo](https://img.shields.io/github/issues-search/astronomy-commons/hats-import?color=purple&label=Good%20first%20issues&query=is%3Aopen%20label%3A%22good%20first%20issue%22)](https://github.com/astronomy-commons/hats-import/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) -See the [contribution guide](https://hipscat-import.readthedocs.io/en/stable/guide/contributing.html) +See the [contribution guide](https://hats-import.readthedocs.io/en/stable/guide/contributing.html) for complete installation instructions and contribution best practices. ## Acknowledgements diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index fbe36f21..1c4c537e 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -3,9 +3,9 @@ // you know what you are doing. "version": 1, // The name of the project being benchmarked. - "project": "hipscat-import", + "project": "hats-import", // The project's homepage. - "project_url": "https://github.com/astronomy-commons/hipscat-import", + "project_url": "https://github.com/astronomy-commons/hats-import", // The URL or local path of the source code repository for the // project being benchmarked. "repo": "..", @@ -32,7 +32,7 @@ // variable. "environment_type": "virtualenv", // the base URL to show a commit for the project. - "show_commit_url": "https://github.com/astronomy-commons/hipscat-import/commit/", + "show_commit_url": "https://github.com/astronomy-commons/hats-import/commit/", // The Pythons you'd like to test against. If not provided, defaults // to the current version of Python used to run `asv`. "pythons": [ diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py index 061dc651..86f36e98 100644 --- a/benchmarks/benchmarks.py +++ b/benchmarks/benchmarks.py @@ -3,8 +3,8 @@ import numpy as np -from hipscat_import.catalog.resume_plan import ResumePlan -from hipscat_import.catalog.sparse_histogram import SparseHistogram +from hats_import.catalog.resume_plan import ResumePlan +from hats_import.catalog.sparse_histogram import SparseHistogram class BinningSuite: diff --git a/docs/catalogs/arguments.rst b/docs/catalogs/arguments.rst index dc3f1b6e..fa34ff76 100644 --- a/docs/catalogs/arguments.rst +++ b/docs/catalogs/arguments.rst @@ -9,7 +9,7 @@ A minimal arguments block will look something like: .. code-block:: python - from hipscat_import.catalog.arguments import ImportArguments + from hats_import.catalog.arguments import ImportArguments args = ImportArguments( sort_columns="ObjectID", @@ -25,8 +25,8 @@ A minimal arguments block will look something like: More details on each of these parameters is provided in sections below. For the curious, see the API documentation for -:py:class:`hipscat_import.catalog.arguments.ImportArguments`, and its superclass -:py:class:`hipscat_import.runtime_arguments.RuntimeArguments`. +:py:class:`hats_import.catalog.arguments.ImportArguments`, and its superclass +:py:class:`hats_import.runtime_arguments.RuntimeArguments`. Pipeline setup ------------------------------------------------------------------------------- @@ -52,7 +52,7 @@ to the pipeline, ignoring the above arguments. This would look like: .. code-block:: python from dask.distributed import Client - from hipscat_import.pipeline import pipeline_with_client + from hats_import.pipeline import pipeline_with_client args = ... # ImportArguments() with Client('scheduler:port') as client: @@ -63,7 +63,7 @@ potentially avoid some python threading issues with dask: .. code-block:: python - from hipscat_import.pipeline import pipeline + from hats_import.pipeline import pipeline def import_pipeline(): args = ... @@ -88,14 +88,14 @@ files are found, we will restore the pipeline's previous progress. If you want to start the pipeline from scratch you can simply set `resume=False`. Alternatively, go to the temp directory you've specified and remove any intermediate -files created by the previous runs of the ``hipscat-import`` pipeline. You should also +files created by the previous runs of the ``hats-import`` pipeline. You should also remove the output directory if it has any content. The resume argument performs these cleaning operations automatically for you. Reading input files ------------------------------------------------------------------------------- -Catalog import reads through a list of files and converts them into a hipscatted catalog. +Catalog import reads through a list of files and converts them into a hats-sharded catalog. Which files? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -134,7 +134,7 @@ to parse a whitespace separated file. Otherwise, you can use a short string to specify an existing file reader type e.g. ``file_reader="csv"``. You can find the full API documentation for -:py:class:`hipscat_import.catalog.file_readers.InputReader` +:py:class:`hats_import.catalog.file_readers.InputReader` .. code-block:: python @@ -150,13 +150,6 @@ You can find the full API documentation for smaller_table = filter_nonsense(smaller_table) yield smaller_table.to_pandas() - def provenance_info(self) -> dict: - provenance_info = { - "input_reader_type": "StarrReader", - "chunksize": self.chunksize, - } - return provenance_info - ... args = ImportArguments( @@ -206,18 +199,18 @@ Which fields? Specify the ``ra_column`` and ``dec_column`` for the dataset. -There are two fields that we require in order to make a valid hipscatted +There are two fields that we require in order to make a valid hats-sharded catalog, the right ascension and declination. At this time, this is the only supported system for celestial coordinates. -If you're importing data that has previously been hipscatted, you may use -``use_hipscat_index = True``. This will use that previously compused hipscat spatial +If you're importing data that has previously been hats-sharded, you may use +``use_healpix_29 = True``. This will use that previously computed hats spatial index as the position, instead of ra/dec. Healpix order and thresholds ------------------------------------------------------------------------------- -When creating a new catalog through the hipscat-import process, we try to +When creating a new catalog through the hats-import process, we try to create partitions with approximately the same number of rows per partition. This isn't perfect, because the sky is uneven, but we still try to create smaller-area pixels in more dense areas, and larger-area pixels in less dense @@ -322,19 +315,19 @@ How? You may want to tweak parameters of the final catalog output, and we have helper arguments for a few of those. -``add_hipscat_index`` - ``bool`` - whether or not to add the hipscat spatial index -as a column in the resulting catalog. The ``_hipscat_index`` field is designed to make many +``add_healpix_29`` - ``bool`` - whether or not to add the hats spatial index +as a column in the resulting catalog. The ``_healpix_29`` field is designed to make many dask operations more performant, but if you do not intend to publish your dataset and do not intend to use dask, then you can suppress generation of this column to save a little space in your final disk usage. -The ``_hipscat_index`` uses a high healpix order and a uniqueness counter to create +The ``_healpix_29`` uses a high healpix order to create values that can order all points in the sky, according to a nested healpix scheme. ``sort_columns`` - ``str`` - column for survey identifier, or other sortable column. If sorting by multiple columns, they should be comma-separated. -If ``add_hipscat_index=True``, this sorting will be used to resolve the -index counter within the same higher-order pixel space. +If ``add_healpix_29=True``, ``_healpix_29`` will be the primary sort key, but the +provided sorting will be used for any rows within the same higher-order pixel space. ``use_schema_file`` - ``str`` - path to a parquet file with schema metadata. This will be used for column metadata when writing the files, if specified. @@ -346,8 +339,6 @@ parquet files with the catalog data, and will only generate root-level metadata files representing the full statistics of the final catalog. This can be useful when probing the import process for effectiveness on processing a target dataset. -``epoch`` - ``str`` - astronomical epoch for the data. defaults to ``"J2000"`` - ``catalog_type`` - ``"object"`` or ``"source"``. Indicates the level of catalog data, using the LSST nomenclature: diff --git a/docs/catalogs/public/allwise.rst b/docs/catalogs/public/allwise.rst index edf9b4e2..0daa246d 100644 --- a/docs/catalogs/public/allwise.rst +++ b/docs/catalogs/public/allwise.rst @@ -32,9 +32,9 @@ Example import import pandas as pd - import hipscat_import.pipeline as runner - from hipscat_import.catalog.arguments import ImportArguments - from hipscat_import.catalog.file_readers import CsvReader + import hats_import.pipeline as runner + from hats_import.catalog.arguments import ImportArguments + from hats_import.catalog.file_readers import CsvReader # Load the column names and types from a side file. type_frame = pd.read_csv("allwise_types.csv") diff --git a/docs/catalogs/public/neowise.rst b/docs/catalogs/public/neowise.rst index 4a21fd8c..5f7657b3 100644 --- a/docs/catalogs/public/neowise.rst +++ b/docs/catalogs/public/neowise.rst @@ -32,9 +32,9 @@ Example import import pandas as pd - import hipscat_import.pipeline as runner - from hipscat_import.catalog.arguments import ImportArguments - from hipscat_import.catalog.file_readers import CsvReader + import hats_import.pipeline as runner + from hats_import.catalog.arguments import ImportArguments + from hats_import.catalog.file_readers import CsvReader # Load the column names and types from a side file. type_frame = pd.read_csv("neowise_types.csv") diff --git a/docs/catalogs/public/panstarrs.rst b/docs/catalogs/public/panstarrs.rst index c5141d8f..edcac8d6 100644 --- a/docs/catalogs/public/panstarrs.rst +++ b/docs/catalogs/public/panstarrs.rst @@ -30,9 +30,9 @@ Example import of objects (otmo) import pandas as pd - import hipscat_import.pipeline as runner - from hipscat_import.catalog.arguments import ImportArguments - from hipscat_import.catalog.file_readers import CsvReader + import hats_import.pipeline as runner + from hats_import.catalog.arguments import ImportArguments + from hats_import.catalog.file_readers import CsvReader # Load the column names and types from a side file. type_frame = pd.read_csv("ps1_otmo_types.csv") diff --git a/docs/catalogs/public/sdss.rst b/docs/catalogs/public/sdss.rst index 6ad74cc7..fb342644 100644 --- a/docs/catalogs/public/sdss.rst +++ b/docs/catalogs/public/sdss.rst @@ -64,8 +64,8 @@ Example import .. code-block:: python - from hipscat_import.catalog.arguments import ImportArguments - import hipscat_import.pipeline as runner + from hats_import.catalog.arguments import ImportArguments + import hats_import.pipeline as runner args = ImportArguments( output_artifact_name="sdss_dr16q", diff --git a/docs/catalogs/public/tic.rst b/docs/catalogs/public/tic.rst index 9376347e..1902cb19 100644 --- a/docs/catalogs/public/tic.rst +++ b/docs/catalogs/public/tic.rst @@ -30,9 +30,9 @@ Example import import pandas as pd - import hipscat_import.pipeline as runner - from hipscat_import.catalog.arguments import ImportArguments - from hipscat_import.catalog.file_readers import CsvReader + import hats_import.pipeline as runner + from hats_import.catalog.arguments import ImportArguments + from hats_import.catalog.file_readers import CsvReader type_frame = pd.read_csv("tic_types.csv") type_map = dict(zip(type_frame["name"], type_frame["type"])) diff --git a/docs/catalogs/public/zubercal.rst b/docs/catalogs/public/zubercal.rst index d2a9e1c9..97835dc7 100644 --- a/docs/catalogs/public/zubercal.rst +++ b/docs/catalogs/public/zubercal.rst @@ -32,9 +32,9 @@ Challenges with this data set .. code-block:: python - import hipscat_import.pipeline as runner - from hipscat_import.catalog.arguments import ImportArguments - from hipscat_import.catalog.file_readers import ParquetReader + import hats_import.pipeline as runner + from hats_import.catalog.arguments import ImportArguments + from hats_import.catalog.file_readers import ParquetReader import pyarrow.parquet as pq import pyarrow as pa import re diff --git a/docs/catalogs/temp_files.rst b/docs/catalogs/temp_files.rst index 451bc990..a752084d 100644 --- a/docs/catalogs/temp_files.rst +++ b/docs/catalogs/temp_files.rst @@ -1,7 +1,7 @@ Temporary files and disk usage =============================================================================== -This page aims to characterize intermediate files created by the hipscat-import +This page aims to characterize intermediate files created by the hats-import catalog creation process. Most users are going to be ok with setting the ``tmp_dir`` and not thinking much more about it. @@ -90,7 +90,7 @@ Some more explanation: What's happening when ------------------------------------------------------------------------------- -The hipscat-import catalog creation process generates a lot of temporary files. Some find this +The hats-import catalog creation process generates a lot of temporary files. Some find this surprising, so we try to provide a narrative of what's happening and why. Planning stage @@ -159,7 +159,7 @@ This is when storage shifts from intermediate files to the real output files. Finishing stage ............................................................................... -Here, we will write out a few additional final files (e.g. ``catalog_info.json``, ``_metadata``). +Here, we will write out a few additional final files (e.g. ``properties``, ``_metadata``). Additionally, we will clean up any straggling intermediate resume files. This includes all text log files, and the summed histogram file. After this stage, we should have zero intermediate files. @@ -196,10 +196,10 @@ final catalog can be very different from the on-disk size of the input files. In our internal testing, we converted a number of different kinds of catalogs, and share some of the results with you, to give some suggestion of the disk requirements -you may face when converting your own catalogs to hipscat format. +you may face when converting your own catalogs to hats format. ============= =============== =========== =============== ========================= -Catalog Input size (-h) Input size Hipscatted size Ratio +Catalog Input size (-h) Input size HATS size Ratio ============= =============== =========== =============== ========================= allwise 1.2T 1196115700 310184460 0.26 (a lot smaller) neowise 3.9T 4177447284 4263269112 1.02 (about the same) @@ -213,4 +213,4 @@ Notes: - allwise, neowise, and tic were all originally compressed CSV files. - sdss was originally a series of fits files - zubercal was originally 500k parquet files, and is reduced in the example to - around 70k hipscat parquet files. + around 70k hats parquet files. diff --git a/docs/conf.py b/docs/conf.py index a41343d2..eaf804a4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,10 +14,10 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = "hipscat-import" +project = "hats-import" copyright = "2023, LINCC Frameworks" author = "LINCC Frameworks" -release = version("hipscat-import") +release = version("hats-import") # for example take major/minor version = ".".join(release.split(".")[:2]) @@ -80,8 +80,8 @@ ## lets us suppress the copy button on select code blocks. copybutton_selector = "div:not(.no-copybutton) > div.highlight > pre" -# Cross-link hipscat documentation from the API reference: +# Cross-link hats documentation from the API reference: # https://docs.readthedocs.io/en/stable/guides/intersphinx.html intersphinx_mapping = { - "hipscat": ("http://hipscat.readthedocs.io/en/stable/", None), + "hats": ("http://hats.readthedocs.io/en/stable/", None), } diff --git a/docs/guide/contact.rst b/docs/guide/contact.rst index 48fa9647..5645b658 100644 --- a/docs/guide/contact.rst +++ b/docs/guide/contact.rst @@ -6,7 +6,7 @@ We at LINCC Frameworks pride ourselves on being a friendly bunch! If you're encountering issues, have some gnarly dataset, have ideas for making our products better, or pretty much anything else, reach out! -* Open an issue in our github repo for hipscat-import - * https://github.com/astronomy-commons/hipscat-import/issues/new +* Open an issue in our github repo for hats-import + * https://github.com/astronomy-commons/hats-import/issues/new * If you're on LSSTC slack, so are we! `#lincc-frameworks-qa `_ \ No newline at end of file diff --git a/docs/guide/contributing.rst b/docs/guide/contributing.rst index b2e1f545..7aa478a8 100644 --- a/docs/guide/contributing.rst +++ b/docs/guide/contributing.rst @@ -1,4 +1,4 @@ -Contributing to hipscat-import +Contributing to hats-import =============================================================================== Find (or make) a new GitHub issue diff --git a/docs/guide/dask_on_ray.rst b/docs/guide/dask_on_ray.rst index a80ade10..53d290d2 100644 --- a/docs/guide/dask_on_ray.rst +++ b/docs/guide/dask_on_ray.rst @@ -8,7 +8,7 @@ See more on Ray's site: https://docs.ray.io/en/latest/ray-more-libs/dask-on-ray.html -How to use in hipscat-import pipelines +How to use in hats-import pipelines ------------------------------------------------------------------------------- Install ray @@ -27,7 +27,7 @@ You should also disable ray when you're done, just to clean things up. from dask.distributed import Client from ray.util.dask import disable_dask_on_ray, enable_dask_on_ray - from hipscat_import.pipeline import pipeline_with_client + from hats_import.pipeline import pipeline_with_client with ray.init( num_cpus=args.dask_n_workers, diff --git a/docs/guide/index_table.rst b/docs/guide/index_table.rst index eb816bf2..9bff8ea1 100644 --- a/docs/guide/index_table.rst +++ b/docs/guide/index_table.rst @@ -2,7 +2,7 @@ Index Table =============================================================================== This page discusses topics around setting up a pipeline to generate a secondary -index lookup for a field on an existing hipscat catalog on disk. +index lookup for a field on an existing hats catalog on disk. This is useful if you would like to have quick access to rows of your table using a survey-provided unique identifier that is NOT spatially correlated. To find @@ -15,7 +15,7 @@ and where to put the output files. A minimal arguments block will look something .. code-block:: python - from hipscat_import.index.arguments import IndexArguments + from hats_import.index.arguments import IndexArguments args = IndexArguments( input_catalog_path="./my_data/my_catalog", @@ -27,8 +27,8 @@ and where to put the output files. A minimal arguments block will look something More details on each of these parameters is provided in sections below. For the curious, see the API documentation for -:py:class:`hipscat_import.index.arguments.IndexArguments`, -and its superclass :py:class:`hipscat_import.runtime_arguments.RuntimeArguments`. +:py:class:`hats_import.index.arguments.IndexArguments`, +and its superclass :py:class:`hats_import.runtime_arguments.RuntimeArguments`. Dask setup ------------------------------------------------------------------------------- @@ -51,7 +51,7 @@ to the pipeline, ignoring the above arguments. This would look like: .. code-block:: python from dask.distributed import Client - from hipscat_import.pipeline import pipeline_with_client + from hats_import.pipeline import pipeline_with_client args = IndexArguments(...) with Client('scheduler:port') as client: @@ -62,7 +62,7 @@ potentially avoid some python threading issues with dask: .. code-block:: python - from hipscat_import.pipeline import pipeline + from hats_import.pipeline import pipeline def index_pipeline(): args = IndexArguments(...) @@ -75,7 +75,7 @@ Input Catalog ------------------------------------------------------------------------------- For this pipeline, you will need to have already transformed your catalog into -hipscat parquet format. Provide the path to the catalog data with the argument +hats parquet format. Provide the path to the catalog data with the argument ``input_catalog_path``. ``indexing_column`` is required, and is the column that you would like to create @@ -120,7 +120,7 @@ string sorting will be smart enough to collate the various strings appropriately .. code-block:: python - divisions = [f"Gaia DR3 {i}" for i in range(10000, 99999, 12)] + divisions = [f"Gaia DR3 {i}" for i in range(10_000, 99_999, 12)] divisions.append("Gaia DR3 999999988604363776") Getting hints from ``_metadata`` @@ -149,8 +149,8 @@ list along to your ``ImportArguments``! import numpy as np import os - from hipscat.io.parquet_metadata import write_parquet_metadata - from hipscat.io import file_io + from hats.io.parquet_metadata import write_parquet_metadata + from hats.io import file_io ## Specify the catalog and column you're making your index over. input_catalog_path="/data/input_catalog" @@ -249,10 +249,8 @@ arguments for a few of those. ``compute_partition_size`` - ``int`` - partition size used when computing the leaf parquet files. -``include_hipscat_index`` - ``bool`` - whether or not to include the 64-bit -hipscat spatial index in the index table. Defaults to ``True``. It can be -useful to keep this value if the ``_hipscat_index`` is your only unique -identifier, or you intend to re-partition your data. +``include_healpix_29`` - ``bool`` - whether or not to include the 64-bit +hats spatial index in the index table. Defaults to ``True``. ``include_order_pixel`` - ``bool`` - whether to include partitioning columns, ``Norder``, ``Dir``, and ``Npix``. You probably want to keep these! @@ -261,7 +259,7 @@ when trying to use the index table. ``drop_duplicates`` - ``bool`` - drop duplicate occurrences of all fields that are included in the index table. This is enabled by default, but can be -**very** slow. This has an interaction with the above ``include_hipscat_index`` +**very** slow. This has an interaction with the above ``include_healpix_29`` and ``include_order_pixel`` options above. We desribe some common patterns below: - I want to create an index over the target ID in my catalog. There are no @@ -270,8 +268,7 @@ and ``include_order_pixel`` options above. We desribe some common patterns below .. code-block:: python indexing_column="target_id", - # target_id is unique, and I don't need to keep extra data - include_hipscat_index=False, + include_healpix_29=False, # I want to know where my data is in the sky. include_order_pixel=True, # target_id is unique, and I don't need to do extra work to de-duplicate @@ -287,7 +284,7 @@ and ``include_order_pixel`` options above. We desribe some common patterns below indexing_column="target_id", # target_id is NOT unique drop_duplicates=True, - # target_id is NOT unique, but including the _hipscat_index will bloat results - include_hipscat_index=False, + # including the _healpix_29 will bloat results + include_healpix_29=False, # I want to know where my data is in the sky. include_order_pixel=True, diff --git a/docs/guide/margin_cache.rst b/docs/guide/margin_cache.rst index d481b519..613df6cb 100644 --- a/docs/guide/margin_cache.rst +++ b/docs/guide/margin_cache.rst @@ -6,14 +6,14 @@ For more discussion of the whys and hows of margin caches, please see for more information. This page discusses topics around setting up a pipeline to generate a margin -cache from an existing hipscat catalog on disk. +cache from an existing hats catalog on disk. At a minimum, you need arguments that include where to find the input files, and where to put the output files. A minimal arguments block will look something like: .. code-block:: python - from hipscat_import.margin_cache.margin_cache_arguments import MarginCacheArguments + from hats_import.margin_cache.margin_cache_arguments import MarginCacheArguments args = MarginCacheArguments( input_catalog_path="./my_data/my_catalog", @@ -26,8 +26,8 @@ and where to put the output files. A minimal arguments block will look something More details on each of these parameters is provided in sections below. For the curious, see the API documentation for -:py:class:`hipscat_import.margin_cache.margin_cache_arguments.MarginCacheArguments`, -and its superclass :py:class:`hipscat_import.runtime_arguments.RuntimeArguments`. +:py:class:`hats_import.margin_cache.margin_cache_arguments.MarginCacheArguments`, +and its superclass :py:class:`hats_import.runtime_arguments.RuntimeArguments`. Dask setup ------------------------------------------------------------------------------- @@ -50,7 +50,7 @@ to the pipeline, ignoring the above arguments. This would look like: .. code-block:: python from dask.distributed import Client - from hipscat_import.pipeline import pipeline_with_client + from hats_import.pipeline import pipeline_with_client args = MarginCacheArguments(...) with Client('scheduler:port') as client: @@ -61,7 +61,7 @@ potentially avoid some python threading issues with dask: .. code-block:: python - from hipscat_import.pipeline import pipeline + from hats_import.pipeline import pipeline def margin_pipeline(): args = MarginCacheArguments(...) @@ -74,10 +74,10 @@ Input Catalog ------------------------------------------------------------------------------- For this pipeline, you will need to have already transformed your catalog into -hipscat parquet format. Provide the path to the catalog data with the argument +hats parquet format. Provide the path to the catalog data with the argument ``input_catalog_path``. -The input hipscat catalog will provide its own right ascension and declination +The input hats catalog will provide its own right ascension and declination that will be used when computing margin populations. Margin calculation parameters diff --git a/docs/index.rst b/docs/index.rst index 57852de1..01d6bb7f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,25 +1,25 @@ -HiPSCat Import +HATS Import ======================================================================================== -Utility for ingesting large survey data into HiPSCat structure. +Utility for ingesting large survey data into HATS structure. Installation ------------------------------------------------------------------------------- We recommend installing in a virtual environment, like venv or conda. You may -need to install or upgrade versions of dependencies to work with hipscat-import. +need to install or upgrade versions of dependencies to work with hats-import. .. code-block:: console - pip install hipscat-import + pip install hats-import .. tip:: Installing on Mac - ``healpy`` is a very necessary dependency for hipscat libraries at this time, but + ``healpy`` is a very necessary dependency for hats libraries at this time, but native prebuilt binaries for healpy on Apple Silicon Macs `do not yet exist `_, - so it's recommended to install via conda before proceeding to hipscat-import. + so it's recommended to install via conda before proceeding to hats-import. .. code-block:: console @@ -29,7 +29,7 @@ need to install or upgrade versions of dependencies to work with hipscat-import. Setting up a pipeline ------------------------------------------------------------------------------- -For each type of dataset the hipscat-import tool can generate, there is an argument +For each type of dataset the hats-import tool can generate, there is an argument container class that you will need to instantiate and populate with relevant arguments. See dataset-specific notes on arguments: @@ -45,7 +45,7 @@ threading issues with dask: .. code-block:: python from dask.distributed import Client - from hipscat_import.pipeline import pipeline_with_client + from hats_import.pipeline import pipeline_with_client def main(): args = ... diff --git a/docs/notebooks/estimate_pixel_threshold.ipynb b/docs/notebooks/estimate_pixel_threshold.ipynb index 670192bc..0908268e 100755 --- a/docs/notebooks/estimate_pixel_threshold.ipynb +++ b/docs/notebooks/estimate_pixel_threshold.ipynb @@ -11,7 +11,7 @@ "\n", "**Background**\n", "\n", - "When creating a new catalog through the hipscat-import process, we try to create partitions with approximately the same number of rows per partition. This isn't perfect, because the sky is uneven, but we still try to create smaller-area pixels in more dense areas, and larger-area pixels in less dense areas. We use the argument `pixel_threshold` and will split a partition into smaller healpix pixels until the number of rows is smaller than `pixel_threshold`.\n", + "When creating a new catalog through the hats-import process, we try to create partitions with approximately the same number of rows per partition. This isn't perfect, because the sky is uneven, but we still try to create smaller-area pixels in more dense areas, and larger-area pixels in less dense areas. We use the argument `pixel_threshold` and will split a partition into smaller healpix pixels until the number of rows is smaller than `pixel_threshold`.\n", "\n", "We do this to increase parallelization of reads and downstream analysis: if the files are around the same size, and operations on each partition take around the same amount of time, we're not as likely to be waiting on a single process to complete for the whole pipeline to complete.\n", "\n", @@ -19,7 +19,7 @@ "\n", "**Objective**\n", "\n", - "In this notebook, we'll go over *one* strategy for estimating the `pixel_threshold` argument you can use when importing a new catalog into hipscat format.\n", + "In this notebook, we'll go over *one* strategy for estimating the `pixel_threshold` argument you can use when importing a new catalog into hats format.\n", "\n", "This is not guaranteed to give you optimal results, but it could give you some hints toward *better* results." ] @@ -60,10 +60,10 @@ "metadata": {}, "outputs": [], "source": [ - "from hipscat_import.catalog.file_readers import CsvReader\n", + "from hats_import.catalog.file_readers import CsvReader\n", "\n", "### Change this path!!!\n", - "input_file = \"../../tests/hipscat_import/data/small_sky/catalog.csv\"\n", + "input_file = \"../../tests/data/small_sky/catalog.csv\"\n", "\n", "file_reader = CsvReader(chunksize=5_000)\n", "\n", @@ -77,7 +77,7 @@ "source": [ "## Inspect parquet file and metadata\n", "\n", - "Now that we have parsed our survey data into parquet, we can check what the data will look like when it's imported into hipscat format.\n", + "Now that we have parsed our survey data into parquet, we can check what the data will look like when it's imported into hats format.\n", "\n", "If you're just here to get a naive estimate for your pixel threshold, we'll do that first, then take a look at some other parquet characteristics later for the curious." ] @@ -161,7 +161,7 @@ "\n", "Below, we inspect the row and column group metadata to show the compressed size of the fields on disk. The last column, `percent`, show the percent of total size taken up by the column.\n", "\n", - "You *can* use this to inform which columns you keep when importing a catalog into hipscat format. e.g. if some columns are less useful for your science, and take up a lot of space, maybe leave them out!" + "You *can* use this to inform which columns you keep when importing a catalog into hats format. e.g. if some columns are less useful for your science, and take up a lot of space, maybe leave them out!" ] }, { @@ -192,7 +192,7 @@ ], "metadata": { "kernelspec": { - "display_name": "hipscatenv", + "display_name": "hatsenv", "language": "python", "name": "python3" }, diff --git a/docs/notebooks/unequal_schema.ipynb b/docs/notebooks/unequal_schema.ipynb index 3fc39e0a..664b448b 100644 --- a/docs/notebooks/unequal_schema.ipynb +++ b/docs/notebooks/unequal_schema.ipynb @@ -67,11 +67,11 @@ "import os\n", "from dask.distributed import Client\n", "\n", - "from hipscat_import.pipeline import pipeline_with_client\n", - "from hipscat_import.catalog.arguments import ImportArguments\n", - "from hipscat_import.catalog.file_readers import get_file_reader\n", + "from hats_import.pipeline import pipeline_with_client\n", + "from hats_import.catalog.arguments import ImportArguments\n", + "from hats_import.catalog.file_readers import get_file_reader\n", "\n", - "mixed_schema_csv_dir = \"../../tests/hipscat_import/data/mixed_schema\"\n", + "mixed_schema_csv_dir = \"../../tests/data/mixed_schema\"\n", "tmp_path = tempfile.TemporaryDirectory()\n", "\n", "args = ImportArguments(\n", @@ -110,7 +110,7 @@ "source": [ "import pyarrow.parquet as pq\n", "\n", - "mixed_schema_csv_parquet = \"../../tests/hipscat_import/data/mixed_schema/schema.parquet\"\n", + "mixed_schema_csv_parquet = \"../../tests/data/mixed_schema/schema.parquet\"\n", "\n", "parquet_file = pq.ParquetFile(mixed_schema_csv_parquet)\n", "print(parquet_file.schema)" @@ -294,7 +294,7 @@ ], "metadata": { "kernelspec": { - "display_name": "hipscatenv", + "display_name": "hatsenv", "language": "python", "name": "python3" }, diff --git a/docs/requirements.txt b/docs/requirements.txt index 11f126d9..80207354 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -9,4 +9,4 @@ sphinx sphinx-autoapi sphinx-copybutton sphinx-book-theme -git+https://github.com/astronomy-commons/hipscat.git@main \ No newline at end of file +git+https://github.com/astronomy-commons/hats.git@main \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9be58956..72abf996 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "hipscat-import" +name = "hats-import" license = {file = "LICENSE"} readme = "README.md" authors = [ @@ -17,7 +17,7 @@ dynamic = ["version"] dependencies = [ "dask[complete]>=2024.3.0", # Includes dask expressions. "deprecated", - "hipscat >=0.3.8", + "hats >=0.4", "ipykernel", # Support for Jupyter notebooks "numpy", "pandas", @@ -40,7 +40,6 @@ dev = [ "pytest", "pytest-cov", "pytest-timeout", - "mypy", # Used for static type checking of files "ray", # Used for dask-on-ray testing. "types-PyYAML", # type stubs for pyyaml ] @@ -53,10 +52,10 @@ requires = [ build-backend = "setuptools.build_meta" [tool.setuptools_scm] -write_to = "src/hipscat_import/_version.py" +write_to = "src/hats_import/_version.py" [tool.setuptools.package-data] -hipscat_import = ["py.typed"] +hats_import = ["py.typed"] [tool.pytest.ini_options] timeout = 1 @@ -69,8 +68,8 @@ testpaths = [ [tool.coverage.report] omit = [ - "src/hipscat_import/_version.py", # auto-generated - "src/hipscat_import/pipeline.py", # too annoying to test + "src/hats_import/_version.py", # auto-generated + "src/hats_import/pipeline.py", # too annoying to test ] [tool.black] @@ -129,6 +128,6 @@ ignore = [ [tool.coverage.run] omit = [ - "src/hipscat_import/_version.py", # auto-generated - "src/hipscat_import/pipeline.py", # too annoying to test + "src/hats_import/_version.py", # auto-generated + "src/hats_import/pipeline.py", # too annoying to test ] diff --git a/requirements.txt b/requirements.txt index 124b2043..d405b240 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -git+https://github.com/astronomy-commons/hipscat.git@main \ No newline at end of file +git+https://github.com/astronomy-commons/hats.git@main \ No newline at end of file diff --git a/src/.pylintrc b/src/.pylintrc index 25eff61d..4bf8b358 100644 --- a/src/.pylintrc +++ b/src/.pylintrc @@ -280,6 +280,8 @@ ignored-parents= # Maximum number of arguments for function / method. max-args=10 +max-positional-arguments=15 + # Maximum number of attributes for a class (see R0902). max-attributes=20 diff --git a/src/hipscat_import/__init__.py b/src/hats_import/__init__.py similarity index 64% rename from src/hipscat_import/__init__.py rename to src/hats_import/__init__.py index ccdbd851..e8990428 100644 --- a/src/hipscat_import/__init__.py +++ b/src/hats_import/__init__.py @@ -1,4 +1,4 @@ -"""All modules for hipscat-import package""" +"""All modules for hats-import package""" from ._version import __version__ from .runtime_arguments import RuntimeArguments diff --git a/src/hipscat_import/catalog/__init__.py b/src/hats_import/catalog/__init__.py similarity index 100% rename from src/hipscat_import/catalog/__init__.py rename to src/hats_import/catalog/__init__.py diff --git a/src/hipscat_import/catalog/arguments.py b/src/hats_import/catalog/arguments.py similarity index 68% rename from src/hipscat_import/catalog/arguments.py rename to src/hats_import/catalog/arguments.py index 89d0c144..51af2724 100644 --- a/src/hipscat_import/catalog/arguments.py +++ b/src/hats_import/catalog/arguments.py @@ -6,12 +6,12 @@ from pathlib import Path from typing import List -from hipscat.catalog.catalog import CatalogInfo -from hipscat.pixel_math import hipscat_id +from hats.catalog import TableProperties +from hats.pixel_math import spatial_index from upath import UPath -from hipscat_import.catalog.file_readers import InputReader, get_file_reader -from hipscat_import.runtime_arguments import RuntimeArguments, find_input_paths +from hats_import.catalog.file_readers import InputReader, get_file_reader +from hats_import.runtime_arguments import RuntimeArguments, find_input_paths # pylint: disable=too-many-locals,too-many-arguments,too-many-instance-attributes,too-many-branches,too-few-public-methods @@ -20,9 +20,6 @@ class ImportArguments(RuntimeArguments): """Container class for holding partitioning arguments""" - epoch: str = "J2000" - """astronomical epoch for the data. defaults to "J2000" """ - catalog_type: str = "object" """level of catalog data, object (things in the sky) or source (detections)""" input_path: str | Path | UPath | None = None @@ -36,14 +33,14 @@ class ImportArguments(RuntimeArguments): """column for right ascension""" dec_column: str = "dec" """column for declination""" - use_hipscat_index: bool = False - """use an existing hipscat spatial index as the position, instead of ra/dec""" + use_healpix_29: bool = False + """use an existing healpix-based hats spatial index as the position, instead of ra/dec""" sort_columns: str | None = None """column for survey identifier, or other sortable column. if sorting by multiple columns, - they should be comma-separated. if `add_hipscat_index=True`, this sorting will be used to - resolve the counter within the same higher-order pixel space""" - add_hipscat_index: bool = True - """add the hipscat spatial index field alongside the data""" + they should be comma-separated. If `add_healpix_29=True`, `_healpix_29` will be the primary sort key, + but the provided sorting will be used for any rows within the same higher-order pixel space.""" + add_healpix_29: bool = True + """add the healpix-based hats spatial index field alongside the data""" use_schema_file: str | Path | UPath | None = None """path to a parquet file with schema metadata. this will be used for column metadata when writing the files, if specified""" @@ -74,13 +71,6 @@ class ImportArguments(RuntimeArguments): """healpix order to use when mapping. will be ``highest_healpix_order`` unless a positive value is provided for ``constant_healpix_order``""" - delete_intermediate_parquet_files: bool = True - """should we delete the smaller intermediate parquet files generated in the - splitting stage, once the relevant reducing stage is complete?""" - delete_resume_log_files: bool = True - """should we delete task-level done files once each stage is complete? - if False, we will keep all sub-histograms from the mapping stage, and all - done marker files at the end of the pipeline.""" run_stages: List[str] = field(default_factory=list) """list of parallel stages to run. options are ['mapping', 'splitting', 'reducing', 'finishing']. ['planning', 'binning'] stages are not optional. @@ -123,53 +113,34 @@ def _check_arguments(self): if isinstance(self.file_reader, str): self.file_reader = get_file_reader(self.file_reader) - if self.use_hipscat_index: - self.add_hipscat_index = False + if self.use_healpix_29: + self.add_healpix_29 = False if self.sort_columns: - raise ValueError("When using _hipscat_index for position, no sort columns should be added") + raise ValueError("When using _healpix_29 for position, no sort columns should be added") # Basic checks complete - make more checks and create directories where necessary self.input_paths = find_input_paths(self.input_path, "**/*.*", self.input_file_list) - def to_catalog_info(self, total_rows) -> CatalogInfo: + def to_table_properties( + self, total_rows: int, highest_order: int, moc_sky_fraction: float + ) -> TableProperties: """Catalog-type-specific dataset info.""" info = { "catalog_name": self.output_artifact_name, "catalog_type": self.catalog_type, "total_rows": total_rows, - "epoch": self.epoch, - "ra_column": self.ra_column, - "dec_column": self.dec_column, - } - return CatalogInfo(**info) - - def additional_runtime_provenance_info(self) -> dict: - file_reader_info = {"type": self.file_reader} - if isinstance(self.file_reader, InputReader): - file_reader_info = self.file_reader.provenance_info() - return { - "catalog_name": self.output_artifact_name, - "epoch": self.epoch, - "catalog_type": self.catalog_type, - "input_path": self.input_path, - "input_paths": self.input_paths, - "input_file_list": self.input_file_list, "ra_column": self.ra_column, "dec_column": self.dec_column, - "use_hipscat_index": self.use_hipscat_index, - "sort_columns": self.sort_columns, - "constant_healpix_order": self.constant_healpix_order, - "lowest_healpix_order": self.lowest_healpix_order, - "highest_healpix_order": self.highest_healpix_order, - "pixel_threshold": self.pixel_threshold, - "mapping_healpix_order": self.mapping_healpix_order, - "debug_stats_only": self.debug_stats_only, - "file_reader_info": file_reader_info, - } + "hats_cols_sort": self.sort_columns, + "hats_max_rows": self.pixel_threshold, + "hats_order": highest_order, + "moc_sky_fraction": f"{moc_sky_fraction:0.5f}", + } | self.extra_property_dict() + return TableProperties(**info) def check_healpix_order_range( - order, field_name, lower_bound=0, upper_bound=hipscat_id.HIPSCAT_ID_HEALPIX_ORDER + order, field_name, lower_bound=0, upper_bound=spatial_index.SPATIAL_INDEX_ORDER ): """Helper method to check if the ``order`` is within the range determined by the ``lower_bound`` and ``upper_bound``, inclusive. @@ -185,7 +156,7 @@ def check_healpix_order_range( """ if lower_bound < 0: raise ValueError("healpix orders must be positive") - if upper_bound > hipscat_id.HIPSCAT_ID_HEALPIX_ORDER: - raise ValueError(f"healpix order should be <= {hipscat_id.HIPSCAT_ID_HEALPIX_ORDER}") + if upper_bound > spatial_index.SPATIAL_INDEX_ORDER: + raise ValueError(f"healpix order should be <= {spatial_index.SPATIAL_INDEX_ORDER}") if not lower_bound <= order <= upper_bound: raise ValueError(f"{field_name} should be between {lower_bound} and {upper_bound}") diff --git a/src/hipscat_import/catalog/file_readers.py b/src/hats_import/catalog/file_readers.py similarity index 95% rename from src/hipscat_import/catalog/file_readers.py rename to src/hats_import/catalog/file_readers.py index 300717d9..40c05be5 100644 --- a/src/hipscat_import/catalog/file_readers.py +++ b/src/hats_import/catalog/file_readers.py @@ -8,7 +8,7 @@ import pyarrow.parquet as pq from astropy.io import ascii as ascii_reader from astropy.table import Table -from hipscat.io import file_io +from hats.io import file_io from upath import UPath # pylint: disable=too-few-public-methods,too-many-arguments @@ -98,21 +98,6 @@ def read(self, input_file, read_columns=None): DataFrame containing chunk of file info. """ - def provenance_info(self) -> dict: - """Create dictionary of parameters for provenance tracking. - - If any `storage_options` have been provided as kwargs, we will replace the - value with ``REDACTED`` for the purpose of writing to provenance info, as it - may contain user names or API keys. - - Returns: - dictionary with all argument_name -> argument_value as key -> value pairs. - """ - all_args = vars(self) - if "kwargs" in all_args and "storage_options" in all_args["kwargs"]: - all_args["kwargs"]["storage_options"] = "REDACTED" - return {"input_reader_type": type(self).__name__, **vars(self)} - def regular_file_exists(self, input_file, **_kwargs): """Check that the `input_file` points to a single regular file diff --git a/src/hipscat_import/catalog/map_reduce.py b/src/hats_import/catalog/map_reduce.py similarity index 82% rename from src/hipscat_import/catalog/map_reduce.py rename to src/hats_import/catalog/map_reduce.py index 799c3339..82027894 100644 --- a/src/hipscat_import/catalog/map_reduce.py +++ b/src/hats_import/catalog/map_reduce.py @@ -1,20 +1,20 @@ -"""Import a set of non-hipscat files using dask for parallelization""" +"""Import a set of non-hats files using dask for parallelization""" import pickle -import hipscat.pixel_math.healpix_shim as hp +import hats.pixel_math.healpix_shim as hp import numpy as np import pyarrow as pa import pyarrow.parquet as pq -from hipscat import pixel_math -from hipscat.io import file_io, paths -from hipscat.pixel_math.healpix_pixel import HealpixPixel -from hipscat.pixel_math.hipscat_id import HIPSCAT_ID_COLUMN, hipscat_id_to_healpix +from hats import pixel_math +from hats.io import file_io, paths +from hats.pixel_math.healpix_pixel import HealpixPixel +from hats.pixel_math.spatial_index import SPATIAL_INDEX_COLUMN, spatial_index_to_healpix from upath import UPath -from hipscat_import.catalog.resume_plan import ResumePlan -from hipscat_import.catalog.sparse_histogram import SparseHistogram -from hipscat_import.pipeline_resume_plan import get_pixel_cache_directory, print_task_failure +from hats_import.catalog.resume_plan import ResumePlan +from hats_import.catalog.sparse_histogram import SparseHistogram +from hats_import.pipeline_resume_plan import get_pixel_cache_directory, print_task_failure # pylint: disable=too-many-locals,too-many-arguments @@ -39,7 +39,7 @@ def _iterate_input_file( highest_order, ra_column, dec_column, - use_hipscat_index=False, + use_healpix_29=False, read_columns=None, ): """Helper function to handle input file reading and healpix pixel calculation""" @@ -49,11 +49,13 @@ def _iterate_input_file( raise NotImplementedError("No file reader implemented") for chunk_number, data in enumerate(file_reader.read(input_file, read_columns=read_columns)): - if use_hipscat_index: - if data.index.name == HIPSCAT_ID_COLUMN: - mapped_pixels = hipscat_id_to_healpix(data.index, target_order=highest_order) + if use_healpix_29: + if data.index.name == SPATIAL_INDEX_COLUMN: + mapped_pixels = spatial_index_to_healpix(data.index, target_order=highest_order) else: - mapped_pixels = hipscat_id_to_healpix(data[HIPSCAT_ID_COLUMN], target_order=highest_order) + mapped_pixels = spatial_index_to_healpix( + data[SPATIAL_INDEX_COLUMN], target_order=highest_order + ) else: # Set up the pixel data mapped_pixels = hp.ang2pix( @@ -74,13 +76,13 @@ def map_to_pixels( highest_order, ra_column, dec_column, - use_hipscat_index=False, + use_healpix_29=False, ): """Map a file of input objects to their healpix pixels. Args: input_file (UPath): file to read for catalog data. - file_reader (hipscat_import.catalog.file_readers.InputReader): instance of input + file_reader (hats_import.catalog.file_readers.InputReader): instance of input reader that specifies arguments necessary for reading from the input file. resume_path (UPath): where to write resume partial results. mapping_key (str): unique counter for this input file, used @@ -99,8 +101,8 @@ def map_to_pixels( try: histo = SparseHistogram.make_empty(highest_order) - if use_hipscat_index: - read_columns = [HIPSCAT_ID_COLUMN] + if use_healpix_29: + read_columns = [SPATIAL_INDEX_COLUMN] else: read_columns = [ra_column, dec_column] @@ -110,7 +112,7 @@ def map_to_pixels( highest_order, ra_column, dec_column, - use_hipscat_index, + use_healpix_29, read_columns, ): mapped_pixel, count_at_pixel = np.unique(mapped_pixels, return_counts=True) @@ -136,13 +138,13 @@ def split_pixels( cache_shard_path: UPath, resume_path: UPath, alignment_file=None, - use_hipscat_index=False, + use_healpix_29=False, ): """Map a file of input objects to their healpix pixels and split into shards. Args: input_file (UPath): file to read for catalog data. - file_reader (hipscat_import.catalog.file_readers.InputReader): instance + file_reader (hats_import.catalog.file_readers.InputReader): instance of input reader that specifies arguments necessary for reading from the input file. splitting_key (str): unique counter for this input file, used when creating intermediate files @@ -160,16 +162,13 @@ def split_pixels( with open(alignment_file, "rb") as pickle_file: alignment = pickle.load(pickle_file) for chunk_number, data, mapped_pixels in _iterate_input_file( - input_file, pickled_reader_file, highest_order, ra_column, dec_column, use_hipscat_index + input_file, pickled_reader_file, highest_order, ra_column, dec_column, use_healpix_29 ): aligned_pixels = alignment[mapped_pixels] unique_pixels, unique_inverse = np.unique(aligned_pixels, return_inverse=True) for unique_index, [order, pixel, _] in enumerate(unique_pixels): - mapped_indexes = np.where(unique_inverse == unique_index) - data_indexes = data.index[mapped_indexes[0].tolist()] - - filtered_data = data.filter(items=data_indexes, axis=0) + filtered_data = data.iloc[unique_inverse == unique_index] pixel_dir = get_pixel_cache_directory(cache_shard_path, HealpixPixel(order, pixel)) file_io.make_directory(pixel_dir, exist_ok=True) @@ -180,7 +179,7 @@ def split_pixels( filtered_data.to_parquet(output_file.path, index=True, filesystem=output_file.fs) else: filtered_data.to_parquet(output_file.path, index=False, filesystem=output_file.fs) - del filtered_data, data_indexes + del filtered_data ResumePlan.splitting_key_done(tmp_path=resume_path, splitting_key=splitting_key) except Exception as exception: # pylint: disable=broad-exception-caught @@ -199,8 +198,8 @@ def reduce_pixel_shards( ra_column, dec_column, sort_columns: str = "", - use_hipscat_index=False, - add_hipscat_index=True, + use_healpix_29=False, + add_healpix_29=True, delete_input_files=True, use_schema_file="", ): @@ -212,15 +211,15 @@ def reduce_pixel_shards( - ``Norder`` - the healpix order for the pixel - ``Dir`` - the directory part, corresponding to the pixel - ``Npix`` - the healpix pixel - - ``_hipscat_index`` - optional - a spatially-correlated + - ``_healpix_29`` - optional - a spatially-correlated 64-bit index field. - Notes on ``_hipscat_index``: + Notes on ``_healpix_29``: - if we generate the field, we will promote any previous *named* pandas index field(s) to a column with that name. - - see ``hipscat.pixel_math.hipscat_id`` + - see ``hats.pixel_math.spatial_index`` for more in-depth discussion of this field. Args: @@ -235,7 +234,7 @@ def reduce_pixel_shards( for the catalog's final pixel output_path (UPath): where to write the final catalog pixel data sort_columns (str): column for survey identifier, or other sortable column - add_hipscat_index (bool): should we add a _hipscat_index column to + add_healpix_29 (bool): should we add a _healpix_29 column to the resulting parquet file? delete_input_files (bool): should we delete the intermediate files used as input for this method. @@ -281,22 +280,22 @@ def reduce_pixel_shards( dataframe = merged_table.to_pandas() if sort_columns: dataframe = dataframe.sort_values(sort_columns.split(",")) - if add_hipscat_index: + if add_healpix_29: ## If we had a meaningful index before, preserve it as a column. if _has_named_index(dataframe): dataframe = dataframe.reset_index() - dataframe[HIPSCAT_ID_COLUMN] = pixel_math.compute_hipscat_id( + dataframe[SPATIAL_INDEX_COLUMN] = pixel_math.compute_spatial_index( dataframe[ra_column].values, dataframe[dec_column].values, ) - dataframe = dataframe.set_index(HIPSCAT_ID_COLUMN).sort_index() + dataframe = dataframe.set_index(SPATIAL_INDEX_COLUMN).sort_index() - # Adjust the schema to make sure that the _hipscat_index will + # Adjust the schema to make sure that the _healpix_29 will # be saved as a uint64 - elif use_hipscat_index: - if dataframe.index.name != HIPSCAT_ID_COLUMN: - dataframe = dataframe.set_index(HIPSCAT_ID_COLUMN) + elif use_healpix_29: + if dataframe.index.name != SPATIAL_INDEX_COLUMN: + dataframe = dataframe.set_index(SPATIAL_INDEX_COLUMN) dataframe = dataframe.sort_index() dataframe["Norder"] = np.full(rows_written, fill_value=healpix_pixel.order, dtype=np.uint8) @@ -304,7 +303,7 @@ def reduce_pixel_shards( dataframe["Npix"] = np.full(rows_written, fill_value=healpix_pixel.pixel, dtype=np.uint64) if schema: - schema = _modify_arrow_schema(schema, add_hipscat_index) + schema = _modify_arrow_schema(schema, add_healpix_29) dataframe.to_parquet(destination_file.path, schema=schema, filesystem=destination_file.fs) else: dataframe.to_parquet(destination_file.path, filesystem=destination_file.fs) @@ -325,12 +324,12 @@ def reduce_pixel_shards( raise exception -def _modify_arrow_schema(schema, add_hipscat_index): - if add_hipscat_index: +def _modify_arrow_schema(schema, add_healpix_29): + if add_healpix_29: pandas_index_column = schema.get_field_index("__index_level_0__") if pandas_index_column != -1: schema = schema.remove(pandas_index_column) - schema = schema.insert(0, pa.field(HIPSCAT_ID_COLUMN, pa.uint64())) + schema = schema.insert(0, pa.field(SPATIAL_INDEX_COLUMN, pa.int64())) schema = ( schema.append(pa.field("Norder", pa.uint8())) .append(pa.field("Dir", pa.uint64())) diff --git a/src/hipscat_import/catalog/resume_plan.py b/src/hats_import/catalog/resume_plan.py similarity index 95% rename from src/hipscat_import/catalog/resume_plan.py rename to src/hats_import/catalog/resume_plan.py index 03acb7c9..aa7221cb 100644 --- a/src/hipscat_import/catalog/resume_plan.py +++ b/src/hats_import/catalog/resume_plan.py @@ -6,17 +6,17 @@ from dataclasses import dataclass, field from typing import List, Optional, Tuple -import hipscat.pixel_math.healpix_shim as hp +import hats.pixel_math.healpix_shim as hp import numpy as np -from hipscat import pixel_math -from hipscat.io import file_io -from hipscat.pixel_math import empty_histogram -from hipscat.pixel_math.healpix_pixel import HealpixPixel +from hats import pixel_math +from hats.io import file_io +from hats.pixel_math import empty_histogram +from hats.pixel_math.healpix_pixel import HealpixPixel from numpy import frombuffer from upath import UPath -from hipscat_import.catalog.sparse_histogram import SparseHistogram -from hipscat_import.pipeline_resume_plan import PipelineResumePlan +from hats_import.catalog.sparse_histogram import SparseHistogram +from hats_import.pipeline_resume_plan import PipelineResumePlan @dataclass @@ -59,15 +59,7 @@ def __init__( import_args=None, ): if import_args: - super().__init__( - resume=import_args.resume, - progress_bar=import_args.progress_bar, - simple_progress_bar=import_args.simple_progress_bar, - tmp_path=import_args.resume_tmp, - tmp_base_path=import_args.tmp_base_path, - delete_resume_log_files=import_args.delete_resume_log_files, - delete_intermediate_parquet_files=import_args.delete_intermediate_parquet_files, - ) + super().__init__(**import_args.resume_kwargs_dict()) if import_args.debug_stats_only: run_stages = ["mapping", "finishing"] self.input_paths = import_args.input_paths diff --git a/src/hipscat_import/catalog/run_import.py b/src/hats_import/catalog/run_import.py similarity index 81% rename from src/hipscat_import/catalog/run_import.py rename to src/hats_import/catalog/run_import.py index 20ca721f..745a367a 100644 --- a/src/hipscat_import/catalog/run_import.py +++ b/src/hats_import/catalog/run_import.py @@ -1,4 +1,4 @@ -"""Import a set of non-hipscat files using dask for parallelization +"""Import a set of non-hats files using dask for parallelization Methods in this file set up a dask pipeline using futures. The actual logic of the map reduce is in the `map_reduce.py` file. @@ -7,14 +7,14 @@ import os import pickle -import hipscat.io.write_metadata as io -from hipscat.catalog import PartitionInfo -from hipscat.io import paths -from hipscat.io.parquet_metadata import write_parquet_metadata +import hats.io.file_io as io +from hats.catalog import PartitionInfo +from hats.io import paths +from hats.io.parquet_metadata import write_parquet_metadata -import hipscat_import.catalog.map_reduce as mr -from hipscat_import.catalog.arguments import ImportArguments -from hipscat_import.catalog.resume_plan import ResumePlan +import hats_import.catalog.map_reduce as mr +from hats_import.catalog.arguments import ImportArguments +from hats_import.catalog.resume_plan import ResumePlan def run(args, client): @@ -43,7 +43,7 @@ def run(args, client): highest_order=args.mapping_healpix_order, ra_column=args.ra_column, dec_column=args.dec_column, - use_hipscat_index=args.use_hipscat_index, + use_healpix_29=args.use_healpix_29, ) ) resume_plan.wait_for_mapping(futures) @@ -84,7 +84,7 @@ def run(args, client): cache_shard_path=args.tmp_path, resume_path=resume_plan.tmp_path, alignment_file=alignment_file, - use_hipscat_index=args.use_hipscat_index, + use_healpix_29=args.use_healpix_29, ) ) @@ -110,9 +110,9 @@ def run(args, client): ra_column=args.ra_column, dec_column=args.dec_column, sort_columns=args.sort_columns, - add_hipscat_index=args.add_hipscat_index, + add_healpix_29=args.add_healpix_29, use_schema_file=args.use_schema_file, - use_hipscat_index=args.use_hipscat_index, + use_healpix_29=args.use_healpix_29, delete_input_files=args.delete_intermediate_parquet_files, ) ) @@ -121,17 +121,7 @@ def run(args, client): # All done - write out the metadata if resume_plan.should_run_finishing: - with resume_plan.print_progress(total=5, stage_name="Finishing") as step_progress: - catalog_info = args.to_catalog_info(total_rows) - io.write_provenance_info( - catalog_base_dir=args.catalog_path, - dataset_info=catalog_info, - tool_args=args.provenance_info(), - ) - step_progress.update(1) - - io.write_catalog_info(catalog_base_dir=args.catalog_path, dataset_info=catalog_info) - step_progress.update(1) + with resume_plan.print_progress(total=4, stage_name="Finishing") as step_progress: partition_info = PartitionInfo.from_healpix(resume_plan.get_destination_pixels()) partition_info_file = paths.get_partition_info_pointer(args.catalog_path) partition_info.write_to_file(partition_info_file) @@ -145,7 +135,12 @@ def run(args, client): else: partition_info.write_to_metadata_files(args.catalog_path) step_progress.update(1) - io.write_fits_map(args.catalog_path, raw_histogram) + catalog_info = args.to_table_properties( + total_rows, partition_info.get_highest_order(), partition_info.calculate_fractional_coverage() + ) + catalog_info.to_properties_file(args.catalog_path) + step_progress.update(1) + io.write_fits_image(raw_histogram, paths.get_point_map_file_pointer(args.catalog_path)) step_progress.update(1) resume_plan.clean_resume_files() step_progress.update(1) diff --git a/src/hipscat_import/catalog/sparse_histogram.py b/src/hats_import/catalog/sparse_histogram.py similarity index 98% rename from src/hipscat_import/catalog/sparse_histogram.py rename to src/hats_import/catalog/sparse_histogram.py index ac1549ae..0ed130bb 100644 --- a/src/hipscat_import/catalog/sparse_histogram.py +++ b/src/hats_import/catalog/sparse_histogram.py @@ -1,6 +1,6 @@ """Sparse 1-D histogram of healpix pixel counts.""" -import hipscat.pixel_math.healpix_shim as hp +import hats.pixel_math.healpix_shim as hp import numpy as np from scipy.sparse import csc_array, load_npz, save_npz, sparray diff --git a/src/hats_import/hipscat_conversion/__init__.py b/src/hats_import/hipscat_conversion/__init__.py new file mode 100644 index 00000000..c0086edf --- /dev/null +++ b/src/hats_import/hipscat_conversion/__init__.py @@ -0,0 +1,4 @@ +"""Convert a hipscatted catalog into a HATS catalog, with appropriate metadata/properties.""" + +from .arguments import ConversionArguments +from .run_conversion import run diff --git a/src/hats_import/hipscat_conversion/arguments.py b/src/hats_import/hipscat_conversion/arguments.py new file mode 100644 index 00000000..34e8fc33 --- /dev/null +++ b/src/hats_import/hipscat_conversion/arguments.py @@ -0,0 +1,28 @@ +"""Utility to hold all arguments required throughout hipscat -> hats conversion""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from hats.io import file_io +from upath import UPath + +from hats_import.runtime_arguments import RuntimeArguments + + +@dataclass +class ConversionArguments(RuntimeArguments): + """Data class for holding conversion arguments. Mostly just inheriting from RuntimeArguments""" + + ## Input + input_catalog_path: str | Path | UPath | None = None + + def __post_init__(self): + self._check_arguments() + + def _check_arguments(self): + super()._check_arguments() + if not self.input_catalog_path: + raise ValueError("input_catalog_path is required") + self.input_catalog_path = file_io.get_upath(self.input_catalog_path) diff --git a/src/hats_import/hipscat_conversion/run_conversion.py b/src/hats_import/hipscat_conversion/run_conversion.py new file mode 100644 index 00000000..be64835c --- /dev/null +++ b/src/hats_import/hipscat_conversion/run_conversion.py @@ -0,0 +1,177 @@ +"""Convert a hipscatted catalog into a HATS catalog, with appropriate metadata/properties.""" + +import json +import tempfile +from typing import no_type_check + +import hats.pixel_math.healpix_shim as hp +import numpy as np +import pyarrow.parquet as pq +from dask.distributed import as_completed, get_worker +from dask.distributed import print as dask_print +from hats.catalog import CatalogType, PartitionInfo, TableProperties +from hats.io import file_io, parquet_metadata, paths + +import hats_import +from hats_import.hipscat_conversion.arguments import ConversionArguments +from hats_import.pipeline_resume_plan import print_progress +from hats_import.runtime_arguments import _estimate_dir_size + + +@no_type_check +def run(args: ConversionArguments, client): + """Run index creation pipeline.""" + if not args: + raise TypeError("args is required and should be type ConversionArguments") + if not isinstance(args, ConversionArguments): + raise TypeError("args must be type ConversionArguments") + + # Create basic properties, using catalog info, provenance info, and partition_info files + catalog_info = None + with (args.input_catalog_path / "catalog_info.json").open("r", encoding="utf-8") as json_file: + catalog_info = json.load(json_file) + provenance_info = None + with (args.input_catalog_path / "provenance_info.json").open("r", encoding="utf-8") as json_file: + provenance_info = json.load(json_file) + + catalog_type = CatalogType(catalog_info["catalog_type"]) + if catalog_type not in ( + CatalogType.OBJECT, + CatalogType.SOURCE, + CatalogType.MARGIN, + CatalogType.ASSOCIATION, + ): + raise ValueError("Conversion only implemented for object, source, margin, and association tables") + + catalog_info.pop("epoch", None) + catalog_info = catalog_info | args.extra_property_dict() + if "tool_args" in provenance_info: + builder_str = ( + provenance_info["tool_args"]["tool_name"] + + " v" + + provenance_info["tool_args"]["version"] + + " hats-importer conversion v" + + hats_import.__version__ + ) + catalog_info["hats_builder"] = builder_str + if runtime_args := provenance_info["tool_args"].get("runtime_args"): + catalog_info["hats_cols_sort"] = runtime_args.get("sort_columns") + catalog_info["hats_cols_survey_id"] = runtime_args.get("sort_columns") + catalog_info["hats_max_rows"] = runtime_args.get("pixel_threshold") + + partition_info = PartitionInfo.read_from_dir(args.input_catalog_path) + catalog_info["hats_order"] = partition_info.get_highest_order() + + properties = TableProperties(**catalog_info) + + schema = file_io.read_parquet_metadata( + args.input_catalog_path / "_common_metadata" + ).schema.to_arrow_schema() + + futures = [] + for pixel in partition_info.get_healpix_pixels(): + futures.append( + client.submit( + _convert_partition_file, pixel, args, schema, properties.ra_column, properties.dec_column + ) + ) + for future in print_progress( + as_completed(futures), + stage_name="Converting Parquet", + total=len(futures), + use_progress_bar=args.progress_bar, + simple_progress_bar=args.simple_progress_bar, + ): + if future.status == "error": + raise future.exception() + + with print_progress( + total=4, + stage_name="Finishing", + use_progress_bar=args.progress_bar, + simple_progress_bar=args.simple_progress_bar, + ) as step_progress: + total_rows = parquet_metadata.write_parquet_metadata(args.catalog_path) + if total_rows != properties.total_rows: + raise ValueError( + f"Unexpected number of rows (original: {properties.total_rows}" + f" written to parquet: {total_rows})" + ) + step_progress.update(1) + file_io.remove_directory(args.tmp_path, ignore_errors=True) + step_progress.update(1) + ## Update total size with newly-written parquet files. + properties.__pydantic_extra__["hats_estsize"] = int(_estimate_dir_size(args.catalog_path) / 1024) + properties.to_properties_file(args.catalog_path) + partition_info.write_to_file(args.catalog_path / "partition_info.csv") + step_progress.update(1) + _write_nested_fits_map(args.input_catalog_path, args.catalog_path) + step_progress.update(1) + + +def _convert_partition_file(pixel, args, schema, ra_column, dec_column): + try: + # Paths are changed between hipscat and HATS! + input_file = ( + args.input_catalog_path + / f"Norder={pixel.order}" + / f"Dir={pixel.dir}" + / f"Npix={pixel.pixel}.parquet" + ) + + table = pq.read_table(input_file, schema=schema) + num_rows = len(table) + + table = ( + table.drop_columns(["_hipscat_index", "Norder", "Dir", "Npix"]) + .add_column( + 0, + "_healpix_29", + [ + hp.ang2pix( + 2**29, + table[ra_column].to_numpy(), + table[dec_column].to_numpy(), + nest=True, + lonlat=True, + ) + ], + ) + .append_column("Norder", [np.full(num_rows, fill_value=pixel.order, dtype=np.int8)]) + .append_column("Dir", [np.full(num_rows, fill_value=pixel.dir, dtype=np.int64)]) + .append_column("Npix", [np.full(num_rows, fill_value=pixel.pixel, dtype=np.int64)]) + ) + + destination_file = paths.pixel_catalog_file(args.catalog_path, pixel) + destination_file.parent.mkdir(parents=True, exist_ok=True) + pq.write_table(table, destination_file.path, filesystem=destination_file.fs) + except Exception as exception: # pylint: disable=broad-exception-caught + try: + dask_print(" worker address:", get_worker().address) + except Exception: # pylint: disable=broad-exception-caught + pass + dask_print(exception) + + +def _write_nested_fits_map(input_dir, output_dir): + input_file = input_dir / "point_map.fits" + if not input_file.exists(): + return + with tempfile.NamedTemporaryFile() as _tmp_file: + with input_file.open("rb") as _map_file: + map_data = _map_file.read() + _tmp_file.write(map_data) + map_fits_image = hp.read_map(_tmp_file.name, nest=True, h=True) + header_dict = dict(map_fits_image[1]) + if header_dict["ORDERING"] != "NESTED": + map_fits_image = hp.read_map(_tmp_file.name) + else: + map_fits_image = map_fits_image[0] + + output_file = output_dir / "point_map.fits" + with tempfile.NamedTemporaryFile() as _tmp_file: + with output_file.open("wb") as _map_file: + hp.write_map( + _tmp_file.name, map_fits_image, overwrite=True, dtype=np.int32, nest=True, coord="CEL" + ) + _map_file.write(_tmp_file.read()) diff --git a/src/hipscat_import/index/__init__.py b/src/hats_import/index/__init__.py similarity index 54% rename from src/hipscat_import/index/__init__.py rename to src/hats_import/index/__init__.py index 008c9952..f59b54e8 100644 --- a/src/hipscat_import/index/__init__.py +++ b/src/hats_import/index/__init__.py @@ -1,4 +1,4 @@ -"""Create performance index for a single column of an already-hipscatted catalog""" +"""Create performance index for a single column of an already-hats-sharded catalog""" from .arguments import IndexArguments from .map_reduce import create_index diff --git a/src/hipscat_import/index/arguments.py b/src/hats_import/index/arguments.py similarity index 73% rename from src/hipscat_import/index/arguments.py rename to src/hats_import/index/arguments.py index 1fb6fa6b..aba9a700 100644 --- a/src/hipscat_import/index/arguments.py +++ b/src/hats_import/index/arguments.py @@ -6,12 +6,11 @@ from pathlib import Path from typing import List, Optional -from hipscat.catalog import Catalog -from hipscat.catalog.index.index_catalog_info import IndexCatalogInfo -from hipscat.io.validation import is_valid_catalog +from hats.catalog import Catalog, TableProperties +from hats.io.validation import is_valid_catalog from upath import UPath -from hipscat_import.runtime_arguments import RuntimeArguments +from hats_import.runtime_arguments import RuntimeArguments @dataclass @@ -25,8 +24,8 @@ class IndexArguments(RuntimeArguments): extra_columns: List[str] = field(default_factory=list) ## Output - include_hipscat_index: bool = True - """Include the hipscat spatial partition index.""" + include_healpix_29: bool = True + """Include the healpix-based hats spatial index.""" include_order_pixel: bool = True """Include partitioning columns, Norder, Dir, and Npix. You probably want to keep these!""" include_radec: bool = False @@ -57,12 +56,12 @@ def _check_arguments(self): if not self.indexing_column: raise ValueError("indexing_column is required") - if not self.include_hipscat_index and not self.include_order_pixel: - raise ValueError("At least one of include_hipscat_index or include_order_pixel must be True") + if not self.include_healpix_29 and not self.include_order_pixel: + raise ValueError("At least one of include_healpix_29 or include_order_pixel must be True") if not is_valid_catalog(self.input_catalog_path): raise ValueError("input_catalog_path not a valid catalog") - self.input_catalog = Catalog.read_from_hipscat(catalog_path=self.input_catalog_path) + self.input_catalog = Catalog.read_hats(catalog_path=self.input_catalog_path) if self.include_radec: catalog_info = self.input_catalog.catalog_info self.extra_columns.extend([catalog_info.ra_column, catalog_info.dec_column]) @@ -82,24 +81,15 @@ def _check_arguments(self): if self.compute_partition_size < 100_000: raise ValueError("compute_partition_size must be at least 100_000") - def to_catalog_info(self, total_rows) -> IndexCatalogInfo: + def to_table_properties(self, total_rows: int) -> TableProperties: """Catalog-type-specific dataset info.""" info = { "catalog_name": self.output_artifact_name, - "total_rows": total_rows, "catalog_type": "index", - "primary_catalog": self.input_catalog_path, + "total_rows": total_rows, + "primary_catalog": str(self.input_catalog_path), "indexing_column": self.indexing_column, "extra_columns": self.extra_columns, - } - return IndexCatalogInfo(**info) + } | self.extra_property_dict() - def additional_runtime_provenance_info(self) -> dict: - return { - "input_catalog_path": self.input_catalog_path, - "indexing_column": self.indexing_column, - "extra_columns": self.extra_columns, - "include_hipscat_index": self.include_hipscat_index, - "include_order_pixel": self.include_order_pixel, - "include_radec": self.include_radec, - } + return TableProperties(**info) diff --git a/src/hipscat_import/index/map_reduce.py b/src/hats_import/index/map_reduce.py similarity index 82% rename from src/hipscat_import/index/map_reduce.py rename to src/hats_import/index/map_reduce.py index 8bba30ba..fef062a7 100644 --- a/src/hipscat_import/index/map_reduce.py +++ b/src/hats_import/index/map_reduce.py @@ -1,12 +1,12 @@ -"""Create columnar index of hipscat table using dask for parallelization""" +"""Create columnar index of hats table using dask for parallelization""" import dask.dataframe as dd import numpy as np -from hipscat.io import file_io, paths -from hipscat.pixel_math.hipscat_id import HIPSCAT_ID_COLUMN +from hats.io import file_io, paths +from hats.pixel_math.spatial_index import SPATIAL_INDEX_COLUMN -def read_leaf_file(input_file, include_columns, include_hipscat_index, drop_duplicates, schema): +def read_leaf_file(input_file, include_columns, include_healpix_29, drop_duplicates, schema): """Mapping function called once per input file. Reads the leaf parquet file, and returns with appropriate columns and duplicates dropped.""" @@ -18,8 +18,8 @@ def read_leaf_file(input_file, include_columns, include_hipscat_index, drop_dupl ) data = data.reset_index() - if not include_hipscat_index: - data = data.drop(columns=[HIPSCAT_ID_COLUMN]) + if not include_healpix_29: + data = data.drop(columns=[SPATIAL_INDEX_COLUMN]) if drop_duplicates: data = data.drop_duplicates() @@ -35,7 +35,7 @@ def create_index(args, client): if args.include_order_pixel: include_columns.extend(["Norder", "Dir", "Npix"]) - index_dir = file_io.get_upath(args.catalog_path / "index") + index_dir = file_io.get_upath(args.catalog_path / "dataset" / "index") data = dd.from_map( read_leaf_file, @@ -44,7 +44,7 @@ def create_index(args, client): for pixel in args.input_catalog.get_healpix_pixels() ], include_columns=include_columns, - include_hipscat_index=args.include_hipscat_index, + include_healpix_29=args.include_healpix_29, drop_duplicates=args.drop_duplicates, schema=args.input_catalog.schema, ) diff --git a/src/hipscat_import/index/run_index.py b/src/hats_import/index/run_index.py similarity index 52% rename from src/hipscat_import/index/run_index.py rename to src/hats_import/index/run_index.py index fc324af2..2547e883 100644 --- a/src/hipscat_import/index/run_index.py +++ b/src/hats_import/index/run_index.py @@ -1,10 +1,10 @@ -"""Create columnar index of hipscat table using dask for parallelization""" +"""Create columnar index of hats table using dask for parallelization""" -from hipscat.io import file_io, parquet_metadata, write_metadata +from hats.io import file_io, parquet_metadata -import hipscat_import.index.map_reduce as mr -from hipscat_import.index.arguments import IndexArguments -from hipscat_import.pipeline_resume_plan import print_progress +import hats_import.index.map_reduce as mr +from hats_import.index.arguments import IndexArguments +from hats_import.pipeline_resume_plan import print_progress def run(args, client): @@ -17,19 +17,13 @@ def run(args, client): # All done - write out the metadata with print_progress( - total=4, + total=3, stage_name="Finishing", use_progress_bar=args.progress_bar, simple_progress_bar=args.simple_progress_bar, ) as step_progress: - index_catalog_info = args.to_catalog_info(int(rows_written)) - write_metadata.write_provenance_info( - catalog_base_dir=args.catalog_path, - dataset_info=index_catalog_info, - tool_args=args.provenance_info(), - ) - step_progress.update(1) - write_metadata.write_catalog_info(catalog_base_dir=args.catalog_path, dataset_info=index_catalog_info) + index_catalog_info = args.to_table_properties(int(rows_written)) + index_catalog_info.to_properties_file(args.catalog_path) step_progress.update(1) file_io.remove_directory(args.tmp_path, ignore_errors=True) step_progress.update(1) diff --git a/src/hipscat_import/margin_cache/__init__.py b/src/hats_import/margin_cache/__init__.py similarity index 100% rename from src/hipscat_import/margin_cache/__init__.py rename to src/hats_import/margin_cache/__init__.py diff --git a/src/hipscat_import/margin_cache/margin_cache.py b/src/hats_import/margin_cache/margin_cache.py similarity index 80% rename from src/hipscat_import/margin_cache/margin_cache.py rename to src/hats_import/margin_cache/margin_cache.py index c9815139..4042cde1 100644 --- a/src/hipscat_import/margin_cache/margin_cache.py +++ b/src/hats_import/margin_cache/margin_cache.py @@ -1,15 +1,15 @@ -from hipscat.catalog import PartitionInfo -from hipscat.io import file_io, parquet_metadata, paths, write_metadata +from hats.catalog import PartitionInfo +from hats.io import file_io, parquet_metadata, paths -import hipscat_import.margin_cache.margin_cache_map_reduce as mcmr -from hipscat_import.margin_cache.margin_cache_resume_plan import MarginCachePlan +import hats_import.margin_cache.margin_cache_map_reduce as mcmr +from hats_import.margin_cache.margin_cache_resume_plan import MarginCachePlan # pylint: disable=too-many-locals,too-many-arguments def generate_margin_cache(args, client): """Generate a margin cache for a given input catalog. - The input catalog must be in hipscat format. + The input catalog must be in hats format. Args: args (MarginCacheArguments): A valid `MarginCacheArguments` object. @@ -63,17 +63,13 @@ def generate_margin_cache(args, client): partition_info = PartitionInfo.read_from_file(metadata_path) partition_info_file = paths.get_partition_info_pointer(args.catalog_path) partition_info.write_to_file(partition_info_file) - step_progress.update(1) - margin_catalog_info = args.to_catalog_info(int(total_rows)) - write_metadata.write_provenance_info( - catalog_base_dir=args.catalog_path, - dataset_info=margin_catalog_info, - tool_args=args.provenance_info(), - ) - write_metadata.write_catalog_info( - catalog_base_dir=args.catalog_path, dataset_info=margin_catalog_info + margin_catalog_info = args.to_table_properties( + int(total_rows), + partition_info.get_highest_order(), + partition_info.calculate_fractional_coverage(), ) + margin_catalog_info.to_properties_file(args.catalog_path) step_progress.update(1) file_io.remove_directory(args.tmp_path, ignore_errors=True) step_progress.update(1) diff --git a/src/hipscat_import/margin_cache/margin_cache_arguments.py b/src/hats_import/margin_cache/margin_cache_arguments.py similarity index 69% rename from src/hipscat_import/margin_cache/margin_cache_arguments.py rename to src/hats_import/margin_cache/margin_cache_arguments.py index e65f8542..05535942 100644 --- a/src/hipscat_import/margin_cache/margin_cache_arguments.py +++ b/src/hats_import/margin_cache/margin_cache_arguments.py @@ -4,14 +4,13 @@ from pathlib import Path from typing import List -import hipscat.pixel_math.healpix_shim as hp -from hipscat.catalog import Catalog -from hipscat.catalog.margin_cache.margin_cache_catalog_info import MarginCacheCatalogInfo -from hipscat.io.validation import is_valid_catalog -from hipscat.pixel_math.healpix_pixel import HealpixPixel +import hats.pixel_math.healpix_shim as hp +from hats.catalog import Catalog, TableProperties +from hats.io.validation import is_valid_catalog +from hats.pixel_math.healpix_pixel import HealpixPixel from upath import UPath -from hipscat_import.runtime_arguments import RuntimeArguments +from hats_import.runtime_arguments import RuntimeArguments @dataclass @@ -31,15 +30,9 @@ class MarginCacheArguments(RuntimeArguments): fine_filtering: bool = True """should we perform the precise boundary checking? if false, some results may be greater than `margin_threshold` away from the border (but within `margin_order`).""" - delete_intermediate_parquet_files: bool = True - """should we delete the smaller intermediate parquet files generated in the - splitting stage, once the relevant reducing stage is complete?""" - delete_resume_log_files: bool = True - """should we delete task-level done files once each stage is complete? - if False, we will keep all done marker files at the end of the pipeline.""" input_catalog_path: str | Path | UPath | None = None - """the path to the hipscat-formatted input catalog.""" + """the path to the hats-formatted input catalog.""" debug_filter_pixel_list: List[HealpixPixel] = field(default_factory=list) """debug setting. if provided, we will first filter the catalog to the pixels provided. this can be useful for creating a margin over a subset of a catalog.""" @@ -54,7 +47,7 @@ def _check_arguments(self): if not is_valid_catalog(self.input_catalog_path): raise ValueError("input_catalog_path not a valid catalog") - self.catalog = Catalog.read_from_hipscat(self.input_catalog_path) + self.catalog = Catalog.read_hats(self.input_catalog_path) if len(self.debug_filter_pixel_list) > 0: self.catalog = self.catalog.filter_from_pixel_list(self.debug_filter_pixel_list) if len(self.catalog.get_healpix_pixels()) == 0: @@ -76,24 +69,19 @@ def _check_arguments(self): if margin_pixel_mindist * 60.0 < self.margin_threshold: raise ValueError("margin pixels must be larger than margin_threshold") - def to_catalog_info(self, total_rows) -> MarginCacheCatalogInfo: + def to_table_properties( + self, total_rows: int, highest_order: int, moc_sky_fraction: float + ) -> TableProperties: """Catalog-type-specific dataset info.""" info = { "catalog_name": self.output_artifact_name, "total_rows": total_rows, "catalog_type": "margin", - "epoch": self.catalog.catalog_info.epoch, "ra_column": self.catalog.catalog_info.ra_column, "dec_column": self.catalog.catalog_info.dec_column, - "primary_catalog": self.input_catalog_path, + "primary_catalog": str(self.input_catalog_path), "margin_threshold": self.margin_threshold, - } - return MarginCacheCatalogInfo(**info) - - def additional_runtime_provenance_info(self) -> dict: - return { - "input_catalog_path": self.input_catalog_path, - "margin_threshold": self.margin_threshold, - "margin_order": self.margin_order, - "debug_filter_pixel_list": self.debug_filter_pixel_list, - } + "hats_order": highest_order, + "moc_sky_fraction": f"{moc_sky_fraction:0.5f}", + } | self.extra_property_dict() + return TableProperties(**info) diff --git a/src/hipscat_import/margin_cache/margin_cache_map_reduce.py b/src/hats_import/margin_cache/margin_cache_map_reduce.py similarity index 81% rename from src/hipscat_import/margin_cache/margin_cache_map_reduce.py rename to src/hats_import/margin_cache/margin_cache_map_reduce.py index d63d705e..f6ac74a7 100644 --- a/src/hipscat_import/margin_cache/margin_cache_map_reduce.py +++ b/src/hats_import/margin_cache/margin_cache_map_reduce.py @@ -1,15 +1,14 @@ -import hipscat.pixel_math.healpix_shim as hp +import hats.pixel_math.healpix_shim as hp import numpy as np import pandas as pd import pyarrow as pa import pyarrow.dataset as ds -from hipscat import pixel_math -from hipscat.catalog.partition_info import PartitionInfo -from hipscat.io import file_io, paths -from hipscat.pixel_math.healpix_pixel import HealpixPixel +from hats import pixel_math +from hats.io import file_io, paths +from hats.pixel_math.healpix_pixel import HealpixPixel -from hipscat_import.margin_cache.margin_cache_resume_plan import MarginCachePlan -from hipscat_import.pipeline_resume_plan import get_pixel_cache_directory, print_task_failure +from hats_import.margin_cache.margin_cache_resume_plan import MarginCachePlan +from hats_import.pipeline_resume_plan import get_pixel_cache_directory, print_task_failure # pylint: disable=too-many-arguments @@ -112,22 +111,22 @@ def _to_pixel_shard( shard_path = paths.pixel_catalog_file(partition_dir, source_pixel) rename_columns = { - PartitionInfo.METADATA_ORDER_COLUMN_NAME: f"margin_{PartitionInfo.METADATA_ORDER_COLUMN_NAME}", - PartitionInfo.METADATA_DIR_COLUMN_NAME: f"margin_{PartitionInfo.METADATA_DIR_COLUMN_NAME}", - PartitionInfo.METADATA_PIXEL_COLUMN_NAME: f"margin_{PartitionInfo.METADATA_PIXEL_COLUMN_NAME}", + paths.PARTITION_ORDER: paths.MARGIN_ORDER, + paths.PARTITION_DIR: paths.MARGIN_DIR, + paths.PARTITION_PIXEL: paths.MARGIN_PIXEL, } margin_data = margin_data.rename(columns=rename_columns) - margin_data[PartitionInfo.METADATA_ORDER_COLUMN_NAME] = pixel.order - margin_data[PartitionInfo.METADATA_DIR_COLUMN_NAME] = pixel.dir - margin_data[PartitionInfo.METADATA_PIXEL_COLUMN_NAME] = pixel.pixel + margin_data[paths.PARTITION_ORDER] = pixel.order + margin_data[paths.PARTITION_DIR] = pixel.dir + margin_data[paths.PARTITION_PIXEL] = pixel.pixel margin_data = margin_data.astype( { - PartitionInfo.METADATA_ORDER_COLUMN_NAME: np.uint8, - PartitionInfo.METADATA_DIR_COLUMN_NAME: np.uint64, - PartitionInfo.METADATA_PIXEL_COLUMN_NAME: np.uint64, + paths.PARTITION_ORDER: np.uint8, + paths.PARTITION_DIR: np.uint64, + paths.PARTITION_PIXEL: np.uint64, } ) margin_data = margin_data.sort_index() @@ -152,9 +151,9 @@ def reduce_margin_shards( schema = file_io.read_parquet_metadata(original_catalog_metadata).schema.to_arrow_schema() schema = ( - schema.append(pa.field("margin_Norder", pa.uint8())) - .append(pa.field("margin_Dir", pa.uint64())) - .append(pa.field("margin_Npix", pa.uint64())) + schema.append(pa.field(paths.MARGIN_ORDER, pa.uint8())) + .append(pa.field(paths.MARGIN_DIR, pa.uint64())) + .append(pa.field(paths.MARGIN_PIXEL, pa.uint64())) ) data = ds.dataset(shard_dir, format="parquet", schema=schema) full_df = data.to_table().to_pandas() diff --git a/src/hipscat_import/margin_cache/margin_cache_resume_plan.py b/src/hats_import/margin_cache/margin_cache_resume_plan.py similarity index 90% rename from src/hipscat_import/margin_cache/margin_cache_resume_plan.py rename to src/hats_import/margin_cache/margin_cache_resume_plan.py index 000e1ae2..f1239d23 100644 --- a/src/hipscat_import/margin_cache/margin_cache_resume_plan.py +++ b/src/hats_import/margin_cache/margin_cache_resume_plan.py @@ -6,12 +6,12 @@ from typing import List import pandas as pd -from hipscat import pixel_math -from hipscat.io import file_io -from hipscat.pixel_math.healpix_pixel import HealpixPixel +from hats import pixel_math +from hats.io import file_io +from hats.pixel_math.healpix_pixel import HealpixPixel -from hipscat_import.margin_cache.margin_cache_arguments import MarginCacheArguments -from hipscat_import.pipeline_resume_plan import PipelineResumePlan +from hats_import.margin_cache.margin_cache_arguments import MarginCacheArguments +from hats_import.pipeline_resume_plan import PipelineResumePlan @dataclass @@ -29,15 +29,7 @@ class MarginCachePlan(PipelineResumePlan): def __init__(self, args: MarginCacheArguments): if not args.tmp_path: # pragma: no cover (not reachable, but required for mypy) raise ValueError("tmp_path is required") - super().__init__( - resume=args.resume, - progress_bar=args.progress_bar, - simple_progress_bar=args.simple_progress_bar, - tmp_path=args.tmp_path, - tmp_base_path=args.tmp_base_path, - delete_resume_log_files=args.delete_resume_log_files, - delete_intermediate_parquet_files=args.delete_intermediate_parquet_files, - ) + super().__init__(**args.resume_kwargs_dict()) self._gather_plan(args) def _gather_plan(self, args): diff --git a/src/hipscat_import/pipeline.py b/src/hats_import/pipeline.py similarity index 75% rename from src/hipscat_import/pipeline.py rename to src/hats_import/pipeline.py index dd4487d5..f696b4be 100644 --- a/src/hipscat_import/pipeline.py +++ b/src/hats_import/pipeline.py @@ -5,17 +5,17 @@ from dask.distributed import Client -import hipscat_import.catalog.run_import as catalog_runner -import hipscat_import.index.run_index as index_runner -import hipscat_import.margin_cache.margin_cache as margin_runner -import hipscat_import.soap.run_soap as soap_runner -import hipscat_import.verification.run_verification as verification_runner -from hipscat_import.catalog.arguments import ImportArguments -from hipscat_import.index.arguments import IndexArguments -from hipscat_import.margin_cache.margin_cache_arguments import MarginCacheArguments -from hipscat_import.runtime_arguments import RuntimeArguments -from hipscat_import.soap.arguments import SoapArguments -from hipscat_import.verification.arguments import VerificationArguments +import hats_import.catalog.run_import as catalog_runner +import hats_import.index.run_index as index_runner +import hats_import.margin_cache.margin_cache as margin_runner +import hats_import.soap.run_soap as soap_runner +import hats_import.verification.run_verification as verification_runner +from hats_import.catalog.arguments import ImportArguments +from hats_import.index.arguments import IndexArguments +from hats_import.margin_cache.margin_cache_arguments import MarginCacheArguments +from hats_import.runtime_arguments import RuntimeArguments +from hats_import.soap.arguments import SoapArguments +from hats_import.verification.arguments import VerificationArguments # pragma: no cover @@ -62,7 +62,7 @@ def pipeline_with_client(args: RuntimeArguments, client: Client): def _send_failure_email(args: RuntimeArguments, exception: Exception): message = EmailMessage() - message["Subject"] = "hipscat-import failure." + message["Subject"] = "hats-import failure." message["To"] = args.completion_email_address message.set_content( f"output_artifact_name: {args.output_artifact_name}" @@ -77,7 +77,7 @@ def _send_success_email(args): if not args.completion_email_address: return message = EmailMessage() - message["Subject"] = "hipscat-import success." + message["Subject"] = "hats-import success." message["To"] = args.completion_email_address message.set_content(f"output_artifact_name: {args.output_artifact_name}") diff --git a/src/hipscat_import/pipeline_resume_plan.py b/src/hats_import/pipeline_resume_plan.py similarity index 99% rename from src/hipscat_import/pipeline_resume_plan.py rename to src/hats_import/pipeline_resume_plan.py index 95a966af..9e49b15f 100644 --- a/src/hipscat_import/pipeline_resume_plan.py +++ b/src/hats_import/pipeline_resume_plan.py @@ -8,8 +8,8 @@ from dask.distributed import as_completed, get_worker from dask.distributed import print as dask_print -from hipscat.io import file_io -from hipscat.pixel_math.healpix_pixel import HealpixPixel +from hats.io import file_io +from hats.pixel_math.healpix_pixel import HealpixPixel from tqdm.auto import tqdm as auto_tqdm from tqdm.std import tqdm as std_tqdm from upath import UPath @@ -218,7 +218,7 @@ def get_pixel_cache_directory(cache_path, pixel: HealpixPixel): """Create a path for intermediate pixel data. You can use this over the paths.get_pixel_directory method, as it will include the pixel - number in the path. Further, it will just *look* different from a real hipscat + number in the path. Further, it will just *look* different from a real hats path, so it's clearer that it's a temporary directory:: {cache_path}/order_{order}/dir_{dir}/pixel_{pixel}/ diff --git a/src/hipscat_import/py.typed b/src/hats_import/py.typed similarity index 100% rename from src/hipscat_import/py.typed rename to src/hats_import/py.typed diff --git a/src/hipscat_import/runtime_arguments.py b/src/hats_import/runtime_arguments.py similarity index 73% rename from src/hipscat_import/runtime_arguments.py rename to src/hats_import/runtime_arguments.py index eae0ad89..cae7322e 100644 --- a/src/hipscat_import/runtime_arguments.py +++ b/src/hats_import/runtime_arguments.py @@ -4,10 +4,11 @@ import re from dataclasses import dataclass +from datetime import datetime, timezone from importlib.metadata import version from pathlib import Path -from hipscat.io import file_io +from hats.io import file_io from upath import UPath # pylint: disable=too-many-instance-attributes @@ -22,6 +23,11 @@ class RuntimeArguments: """base path where new catalog should be output""" output_artifact_name: str = "" """short, convenient name for the catalog""" + addl_hats_properties: dict | None = None + """Any additional keyword arguments you would like to provide when writing + the `properties` file for the final HATS table. e.g. + {"hats_cols_default":"id, mjd", "hats_cols_survey_id":"unique_id", + "creator_did": "ivo://CDS/P/2MASS/J"}""" ## Execution tmp_dir: str | Path | UPath | None = None @@ -46,6 +52,12 @@ class RuntimeArguments: """number of threads per dask worker""" resume_tmp: str | Path | UPath | None = None """directory for intermediate resume files, when needed. see RTD for more info.""" + delete_intermediate_parquet_files: bool = True + """should we delete the smaller intermediate parquet files generated in the + splitting stage, once the relevant reducing stage is complete?""" + delete_resume_log_files: bool = True + """should we delete task-level done files once each stage is complete? + if False, we will keep all done marker files at the end of the pipeline.""" completion_email_address: str = "" """if provided, send an email to the indicated email address once the @@ -103,37 +115,34 @@ def _check_arguments(self): else: self.resume_tmp = self.tmp_path - def provenance_info(self) -> dict: - """Fill all known information in a dictionary for provenance tracking. - - Returns: - dictionary with all argument_name -> argument_value as key -> value pairs. - """ - runtime_args = { - "catalog_name": self.output_artifact_name, - "output_path": self.output_path, - "output_artifact_name": self.output_artifact_name, - "tmp_dir": self.tmp_dir, - "dask_tmp": self.dask_tmp, - "dask_n_workers": self.dask_n_workers, - "dask_threads_per_worker": self.dask_threads_per_worker, - "catalog_path": self.catalog_path, - "tmp_path": self.tmp_path, + def extra_property_dict(self): + """Generate additional HATS properties for this import run as a dictionary.""" + properties = {} + + properties["hats_builder"] = f"hats-import v{version('hats-import')}" + + now = datetime.now(tz=timezone.utc) + properties["hats_creation_date"] = now.strftime("%Y-%m-%dT%H:%M%Z") + properties["hats_estsize"] = int(_estimate_dir_size(self.catalog_path) / 1024) + properties["hats_release_date"] = "2024-09-18" + properties["hats_version"] = "v0.1" + + if self.addl_hats_properties: + properties = properties | self.addl_hats_properties + return properties + + def resume_kwargs_dict(self): + """Convenience method to convert fields for resume functionality.""" + return { + "resume": self.resume, + "progress_bar": self.progress_bar, + "simple_progress_bar": self.simple_progress_bar, + "tmp_path": self.resume_tmp, + "tmp_base_path": self.tmp_base_path, + "delete_resume_log_files": self.delete_resume_log_files, + "delete_intermediate_parquet_files": self.delete_intermediate_parquet_files, } - runtime_args.update(self.additional_runtime_provenance_info()) - provenance_info = { - "tool_name": "hipscat_import", - "version": version("hipscat-import"), - "runtime_args": runtime_args, - } - - return provenance_info - - def additional_runtime_provenance_info(self): - """Any additional runtime args to be included in provenance info from subclasses""" - return {} - def find_input_paths(input_path="", file_matcher="", input_file_list=None): """Helper method to find input paths, given either a prefix and format, or an @@ -166,3 +175,13 @@ def find_input_paths(input_path="", file_matcher="", input_file_list=None): if len(input_paths) == 0: raise FileNotFoundError("No input files found") return input_paths + + +def _estimate_dir_size(target_dir): + total_size = 0 + for item in target_dir.iterdir(): + if item.is_dir(): + total_size += _estimate_dir_size(item) + else: + total_size += item.stat().st_size + return total_size diff --git a/src/hipscat_import/soap/__init__.py b/src/hats_import/soap/__init__.py similarity index 100% rename from src/hipscat_import/soap/__init__.py rename to src/hats_import/soap/__init__.py diff --git a/src/hipscat_import/soap/arguments.py b/src/hats_import/soap/arguments.py similarity index 54% rename from src/hipscat_import/soap/arguments.py rename to src/hats_import/soap/arguments.py index 2cfe5fe1..1c11c8b6 100644 --- a/src/hipscat_import/soap/arguments.py +++ b/src/hats_import/soap/arguments.py @@ -3,13 +3,12 @@ from dataclasses import dataclass from pathlib import Path -from hipscat.catalog import Catalog -from hipscat.catalog.association_catalog.association_catalog import AssociationCatalogInfo -from hipscat.catalog.catalog_type import CatalogType -from hipscat.io.validation import is_valid_catalog +from hats.catalog import Catalog, TableProperties +from hats.catalog.catalog_type import CatalogType +from hats.io.validation import is_valid_catalog from upath import UPath -from hipscat_import.runtime_arguments import RuntimeArguments +from hats_import.runtime_arguments import RuntimeArguments @dataclass @@ -25,18 +24,9 @@ class SoapArguments(RuntimeArguments): source_object_id_column: str = "" source_id_column: str = "" - resume: bool = True - """if there are existing intermediate resume files, should we - read those and continue to run the pipeline where we left off""" - delete_resume_log_files: bool = True - """should we delete task-level done files once each stage is complete? - if False, we will keep all done marker files at the end of the pipeline.""" write_leaf_files: bool = False """Should we also write out leaf parquet files (e.g. Norder/Dir/Npix.parquet) that represent the full association table""" - delete_intermediate_parquet_files: bool = True - """should we delete the smaller intermediate parquet files generated in the - mapping stage, once the relevant reducing stage is complete?""" compute_partition_size: int = 1_000_000_000 @@ -52,7 +42,7 @@ def _check_arguments(self): if not is_valid_catalog(self.object_catalog_dir): raise ValueError("object_catalog_dir not a valid catalog") - self.object_catalog = Catalog.read_from_hipscat(catalog_path=self.object_catalog_dir) + self.object_catalog = Catalog.read_hats(catalog_path=self.object_catalog_dir) if not self.source_catalog_dir: raise ValueError("source_catalog_dir is required") @@ -61,12 +51,12 @@ def _check_arguments(self): if not is_valid_catalog(self.source_catalog_dir): raise ValueError("source_catalog_dir not a valid catalog") - self.source_catalog = Catalog.read_from_hipscat(catalog_path=self.source_catalog_dir) + self.source_catalog = Catalog.read_hats(catalog_path=self.source_catalog_dir) if self.compute_partition_size < 100_000: raise ValueError("compute_partition_size must be at least 100_000") - def to_catalog_info(self, total_rows) -> AssociationCatalogInfo: + def to_table_properties(self, total_rows=10, highest_order=4, moc_sky_fraction=22 / 7) -> TableProperties: """Catalog-type-specific dataset info.""" info = { "catalog_name": self.output_artifact_name, @@ -74,21 +64,12 @@ def to_catalog_info(self, total_rows) -> AssociationCatalogInfo: "total_rows": total_rows, "primary_column": self.object_id_column, "primary_column_association": "object_id", - "primary_catalog": self.object_catalog_dir, + "primary_catalog": str(self.object_catalog_dir), "join_column": self.source_object_id_column, "join_column_association": "source_id", - "join_catalog": self.source_catalog_dir, + "join_catalog": str(self.source_catalog_dir), "contains_leaf_files": self.write_leaf_files, - } - return AssociationCatalogInfo(**info) - - def additional_runtime_provenance_info(self) -> dict: - return { - "object_catalog_dir": self.object_catalog_dir, - "object_id_column": self.object_id_column, - "source_catalog_dir": self.source_catalog_dir, - "source_object_id_column": self.source_object_id_column, - "source_id_column": self.source_id_column, - "compute_partition_size": self.compute_partition_size, - "write_leaf_files": self.write_leaf_files, - } + "hats_order": highest_order, + "moc_sky_fraction": f"{moc_sky_fraction:0.5f}", + } | self.extra_property_dict() + return TableProperties(**info) diff --git a/src/hipscat_import/soap/map_reduce.py b/src/hats_import/soap/map_reduce.py similarity index 93% rename from src/hipscat_import/soap/map_reduce.py rename to src/hats_import/soap/map_reduce.py index 009d921e..44d8c612 100644 --- a/src/hipscat_import/soap/map_reduce.py +++ b/src/hats_import/soap/map_reduce.py @@ -5,15 +5,15 @@ import numpy as np import pandas as pd import pyarrow.parquet as pq -from hipscat.catalog.association_catalog.partition_join_info import PartitionJoinInfo -from hipscat.io import file_io, paths -from hipscat.io.parquet_metadata import get_healpix_pixel_from_metadata -from hipscat.pixel_math.healpix_pixel import HealpixPixel -from hipscat.pixel_math.healpix_pixel_function import get_pixel_argsort +from hats.catalog.association_catalog.partition_join_info import PartitionJoinInfo +from hats.io import file_io, paths +from hats.io.parquet_metadata import get_healpix_pixel_from_metadata +from hats.pixel_math.healpix_pixel import HealpixPixel +from hats.pixel_math.healpix_pixel_function import get_pixel_argsort -from hipscat_import.pipeline_resume_plan import get_pixel_cache_directory, print_task_failure -from hipscat_import.soap.arguments import SoapArguments -from hipscat_import.soap.resume_plan import SoapPlan +from hats_import.pipeline_resume_plan import get_pixel_cache_directory, print_task_failure +from hats_import.soap.arguments import SoapArguments +from hats_import.soap.resume_plan import SoapPlan def _count_joins_for_object(source_data, source_pixel, object_pixel, soap_args): @@ -84,7 +84,7 @@ def count_joins(soap_args: SoapArguments, source_pixel: HealpixPixel, object_pix If any un-joined source pixels remain, stretch out to neighboring object pixels. Args: - soap_args(`hipscat_import.soap.SoapArguments`): set of arguments for pipeline execution + soap_args(`hats_import.soap.SoapArguments`): set of arguments for pipeline execution source_pixel(HealpixPixel): order and pixel for the source catalog single pixel. object_pixels(List[HealpixPixel]): set of tuples of order and pixel for the partitions of the object catalog to be joined. diff --git a/src/hipscat_import/soap/resume_plan.py b/src/hats_import/soap/resume_plan.py similarity index 90% rename from src/hipscat_import/soap/resume_plan.py rename to src/hats_import/soap/resume_plan.py index a77eb25d..abb04f58 100644 --- a/src/hipscat_import/soap/resume_plan.py +++ b/src/hats_import/soap/resume_plan.py @@ -5,15 +5,15 @@ from dataclasses import dataclass, field from typing import List, Optional, Tuple -import hipscat.pixel_math.healpix_shim as hp +import hats.pixel_math.healpix_shim as hp import numpy as np -from hipscat.catalog import Catalog -from hipscat.io import file_io -from hipscat.pixel_math.healpix_pixel import HealpixPixel -from hipscat.pixel_tree import PixelAlignment, align_trees +from hats.catalog import Catalog +from hats.io import file_io +from hats.pixel_math.healpix_pixel import HealpixPixel +from hats.pixel_tree import PixelAlignment, align_trees -from hipscat_import.pipeline_resume_plan import PipelineResumePlan -from hipscat_import.soap.arguments import SoapArguments +from hats_import.pipeline_resume_plan import PipelineResumePlan +from hats_import.soap.arguments import SoapArguments @dataclass @@ -35,15 +35,7 @@ class SoapPlan(PipelineResumePlan): def __init__(self, args: SoapArguments): if not args.tmp_path: # pragma: no cover (not reachable, but required for mypy) raise ValueError("tmp_path is required") - super().__init__( - resume=args.resume, - progress_bar=args.progress_bar, - simple_progress_bar=args.simple_progress_bar, - tmp_path=args.tmp_path, - tmp_base_path=args.tmp_base_path, - delete_resume_log_files=args.delete_resume_log_files, - delete_intermediate_parquet_files=args.delete_intermediate_parquet_files, - ) + super().__init__(**args.resume_kwargs_dict()) self.gather_plan(args) def gather_plan(self, args): @@ -59,12 +51,12 @@ def gather_plan(self, args): return step_progress.update(1) - self.object_catalog = Catalog.read_from_hipscat(args.object_catalog_dir) + self.object_catalog = Catalog.read_hats(args.object_catalog_dir) source_map_file = file_io.append_paths_to_pointer(self.tmp_path, self.SOURCE_MAP_FILE) if file_io.does_file_or_directory_exist(source_map_file): source_pixel_map = np.load(source_map_file, allow_pickle=True)["arr_0"].item() else: - source_catalog = Catalog.read_from_hipscat(args.source_catalog_dir) + source_catalog = Catalog.read_hats(args.source_catalog_dir) source_pixel_map = source_to_object_map(self.object_catalog, source_catalog) np.savez_compressed(source_map_file, source_pixel_map) self.count_keys = self.get_sources_to_count(source_pixel_map=source_pixel_map) diff --git a/src/hipscat_import/soap/run_soap.py b/src/hats_import/soap/run_soap.py similarity index 76% rename from src/hipscat_import/soap/run_soap.py rename to src/hats_import/soap/run_soap.py index d5f7a0cf..8bd59459 100644 --- a/src/hipscat_import/soap/run_soap.py +++ b/src/hats_import/soap/run_soap.py @@ -3,12 +3,12 @@ The actual logic of the map reduce is in the `map_reduce.py` file. """ -from hipscat.catalog.association_catalog.partition_join_info import PartitionJoinInfo -from hipscat.io import parquet_metadata, paths, write_metadata +from hats.catalog import PartitionInfo, PartitionJoinInfo +from hats.io import parquet_metadata, paths -from hipscat_import.soap.arguments import SoapArguments -from hipscat_import.soap.map_reduce import combine_partial_results, count_joins, reduce_joins -from hipscat_import.soap.resume_plan import SoapPlan +from hats_import.soap.arguments import SoapArguments +from hats_import.soap.map_reduce import combine_partial_results, count_joins, reduce_joins +from hats_import.soap.resume_plan import SoapPlan def run(args, client): @@ -57,14 +57,13 @@ def run(args, client): else: total_rows = combine_partial_results(args.tmp_path, args.catalog_path) step_progress.update(1) - catalog_info = args.to_catalog_info(total_rows) - write_metadata.write_provenance_info( - catalog_base_dir=args.catalog_path, - dataset_info=catalog_info, - tool_args=args.provenance_info(), + partition_info = PartitionInfo.read_from_dir(args.catalog_path) + catalog_info = args.to_table_properties( + total_rows, partition_info.get_highest_order(), partition_info.calculate_fractional_coverage() ) + catalog_info.to_properties_file(args.catalog_path) step_progress.update(1) - write_metadata.write_catalog_info(dataset_info=catalog_info, catalog_base_dir=args.catalog_path) + ## TODO - optionally write out arguments file step_progress.update(1) resume_plan.clean_resume_files() step_progress.update(1) diff --git a/src/hipscat_import/verification/__init__.py b/src/hats_import/verification/__init__.py similarity index 100% rename from src/hipscat_import/verification/__init__.py rename to src/hats_import/verification/__init__.py diff --git a/src/hipscat_import/verification/arguments.py b/src/hats_import/verification/arguments.py similarity index 74% rename from src/hipscat_import/verification/arguments.py rename to src/hats_import/verification/arguments.py index 86c139b1..d17a30ed 100644 --- a/src/hipscat_import/verification/arguments.py +++ b/src/hats_import/verification/arguments.py @@ -6,11 +6,11 @@ from pathlib import Path from typing import List, Optional -from hipscat.catalog import Catalog -from hipscat.io.validation import is_valid_catalog +from hats.catalog import Catalog +from hats.io.validation import is_valid_catalog from upath import UPath -from hipscat_import.runtime_arguments import RuntimeArguments +from hats_import.runtime_arguments import RuntimeArguments @dataclass @@ -39,13 +39,6 @@ def _check_arguments(self): if not self.input_catalog: if not is_valid_catalog(self.input_catalog_path): raise ValueError("input_catalog_path not a valid catalog") - self.input_catalog = Catalog.read_from_hipscat(catalog_path=self.input_catalog_path) + self.input_catalog = Catalog.read_hats(catalog_path=self.input_catalog_path) if not self.input_catalog_path: self.input_catalog_path = self.input_catalog.catalog_path - - def additional_runtime_provenance_info(self) -> dict: - return { - "pipeline": "verification pipeline", - "input_catalog_path": self.input_catalog_path, - "field_distribution_cols": self.field_distribution_cols, - } diff --git a/src/hipscat_import/verification/run_verification.py b/src/hats_import/verification/run_verification.py similarity index 82% rename from src/hipscat_import/verification/run_verification.py rename to src/hats_import/verification/run_verification.py index 2b7d5954..ea623ddf 100644 --- a/src/hipscat_import/verification/run_verification.py +++ b/src/hats_import/verification/run_verification.py @@ -1,6 +1,6 @@ -"""Run pass/fail checks and generate verification report of existing hipscat table.""" +"""Run pass/fail checks and generate verification report of existing hats table.""" -from hipscat_import.verification.arguments import VerificationArguments +from hats_import.verification.arguments import VerificationArguments def run(args): diff --git a/tests/conftest.py b/tests/conftest.py index 7666200f..2d9f12ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -39,7 +39,7 @@ def dask_client(use_ray): def pytest_addoption(parser): """Add command line option to test dask unit tests on ray. - This must live in /tests/conftest.py (not /tests/hipscat-import/conftest.py)""" + This must live in /tests/conftest.py (not /tests/hats-import/conftest.py)""" parser.addoption( "--use_ray", action="store_true", diff --git a/tests/hipscat_import/data/blank/blank.csv b/tests/data/blank/blank.csv similarity index 100% rename from tests/hipscat_import/data/blank/blank.csv rename to tests/data/blank/blank.csv diff --git a/tests/data/generate_data.ipynb b/tests/data/generate_data.ipynb new file mode 100644 index 00000000..d3ba24c9 --- /dev/null +++ b/tests/data/generate_data.ipynb @@ -0,0 +1,145 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Unit test data\n", + "\n", + "This directory contains very small, toy, data sets that are used\n", + "for unit tests.\n", + "\n", + "## Object catalog: small_sky\n", + "\n", + "This \"object catalog\" is 131 randomly generated radec values. \n", + "\n", + "- All radec positions are in the Healpix pixel order 0, pixel 11.\n", + "- IDs are integers from 700-831.\n", + "\n", + "The following are imports and paths that are used throughout the notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import hats_import.pipeline as runner\n", + "from hats_import.catalog.arguments import ImportArguments\n", + "import tempfile\n", + "from pathlib import Path\n", + "from dask.distributed import Client\n", + "\n", + "tmp_path = tempfile.TemporaryDirectory()\n", + "tmp_dir = tmp_path.name\n", + "\n", + "hats_import_dir = \".\"\n", + "client = Client(n_workers=1, threads_per_worker=1, local_directory=tmp_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### small_sky\n", + "\n", + "This \"object catalog\" is 131 randomly generated radec values. \n", + "\n", + "- All radec positions are in the Healpix pixel order 0, pixel 11.\n", + "- IDs are integers from 700-831.\n", + "\n", + "This catalog was generated with the following snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " args = ImportArguments(\n", + " input_path=Path(hats_import_dir) / \"small_sky\",\n", + " output_path=\".\",\n", + " file_reader=\"csv\",\n", + " highest_healpix_order=5,\n", + " output_artifact_name=\"small_sky_object_catalog\",\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + " runner.pipeline_with_client(args, client)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Source catalog: small_sky_source\n", + "\n", + "This \"source catalog\" is 131 detections at each of the 131 objects\n", + "in the \"small_sky\" catalog. These have a random magnitude, MJD, and \n", + "band (selected from ugrizy). The full script that generated the values\n", + "can be found [here](https://github.com/delucchi-cmu/hipscripts/blob/main/twiddling/small_sky_source.py)\n", + "\n", + "The catalog was generated with the following snippet, using raw data \n", + "from the `hats-import` file.\n", + "\n", + "NB: `pixel_threshold=3000` is set just to make sure that we're generating\n", + "a handful of files at various healpix orders." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", + " args = ImportArguments(\n", + " input_path=Path(hats_import_dir) / \"small_sky_source\",\n", + " output_path=\".\",\n", + " file_reader=\"csv\",\n", + " ra_column=\"source_ra\",\n", + " dec_column=\"source_dec\",\n", + " catalog_type=\"source\",\n", + " highest_healpix_order=5,\n", + " pixel_threshold=3000,\n", + " output_artifact_name=\"small_sky_source_catalog\",\n", + " tmp_dir=pipeline_tmp,\n", + " )\n", + " runner.pipeline_with_client(args, client)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client.close()\n", + "tmp_path.cleanup()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/hipscat_import/data/small_sky_object_catalog/Norder=0/Dir=0/Npix=11.parquet b/tests/data/hipscat/small_sky_object_catalog/Norder=0/Dir=0/Npix=11.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_object_catalog/Norder=0/Dir=0/Npix=11.parquet rename to tests/data/hipscat/small_sky_object_catalog/Norder=0/Dir=0/Npix=11.parquet diff --git a/tests/hipscat_import/data/small_sky_object_catalog/_common_metadata b/tests/data/hipscat/small_sky_object_catalog/_common_metadata similarity index 100% rename from tests/hipscat_import/data/small_sky_object_catalog/_common_metadata rename to tests/data/hipscat/small_sky_object_catalog/_common_metadata diff --git a/tests/hipscat_import/data/small_sky_object_catalog/_metadata b/tests/data/hipscat/small_sky_object_catalog/_metadata similarity index 100% rename from tests/hipscat_import/data/small_sky_object_catalog/_metadata rename to tests/data/hipscat/small_sky_object_catalog/_metadata diff --git a/tests/hipscat_import/data/small_sky_object_catalog/catalog_info.json b/tests/data/hipscat/small_sky_object_catalog/catalog_info.json similarity index 100% rename from tests/hipscat_import/data/small_sky_object_catalog/catalog_info.json rename to tests/data/hipscat/small_sky_object_catalog/catalog_info.json diff --git a/tests/hipscat_import/data/small_sky_object_catalog/partition_info.csv b/tests/data/hipscat/small_sky_object_catalog/partition_info.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_object_catalog/partition_info.csv rename to tests/data/hipscat/small_sky_object_catalog/partition_info.csv diff --git a/tests/hipscat_import/data/small_sky_object_catalog/provenance_info.json b/tests/data/hipscat/small_sky_object_catalog/provenance_info.json similarity index 100% rename from tests/hipscat_import/data/small_sky_object_catalog/provenance_info.json rename to tests/data/hipscat/small_sky_object_catalog/provenance_info.json diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=0/Dir=0/Npix=4.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=0/Dir=0/Npix=4.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=0/Dir=0/Npix=4.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=0/Dir=0/Npix=4.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=1/Dir=0/Npix=47.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=1/Dir=0/Npix=47.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=1/Dir=0/Npix=47.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=1/Dir=0/Npix=47.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=176.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=176.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=176.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=176.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=177.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=177.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=177.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=177.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=178.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=178.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=178.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=178.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=179.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=179.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=179.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=179.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=180.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=180.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=180.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=180.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=181.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=181.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=181.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=181.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=182.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=182.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=182.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=182.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=183.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=183.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=183.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=183.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=184.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=184.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=184.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=184.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=185.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=185.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=185.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=185.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=186.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=186.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=186.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=186.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=187.parquet b/tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=187.parquet similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/Norder=2/Dir=0/Npix=187.parquet rename to tests/data/hipscat/small_sky_source_catalog/Norder=2/Dir=0/Npix=187.parquet diff --git a/tests/hipscat_import/data/small_sky_source_catalog/_common_metadata b/tests/data/hipscat/small_sky_source_catalog/_common_metadata similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/_common_metadata rename to tests/data/hipscat/small_sky_source_catalog/_common_metadata diff --git a/tests/hipscat_import/data/small_sky_source_catalog/_metadata b/tests/data/hipscat/small_sky_source_catalog/_metadata similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/_metadata rename to tests/data/hipscat/small_sky_source_catalog/_metadata diff --git a/tests/hipscat_import/data/small_sky_source_catalog/catalog_info.json b/tests/data/hipscat/small_sky_source_catalog/catalog_info.json similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/catalog_info.json rename to tests/data/hipscat/small_sky_source_catalog/catalog_info.json diff --git a/tests/hipscat_import/data/small_sky_source_catalog/partition_info.csv b/tests/data/hipscat/small_sky_source_catalog/partition_info.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/partition_info.csv rename to tests/data/hipscat/small_sky_source_catalog/partition_info.csv diff --git a/tests/hipscat_import/data/small_sky_source_catalog/point_map.fits b/tests/data/hipscat/small_sky_source_catalog/point_map.fits similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/point_map.fits rename to tests/data/hipscat/small_sky_source_catalog/point_map.fits diff --git a/tests/hipscat_import/data/small_sky_source_catalog/provenance_info.json b/tests/data/hipscat/small_sky_source_catalog/provenance_info.json similarity index 100% rename from tests/hipscat_import/data/small_sky_source_catalog/provenance_info.json rename to tests/data/hipscat/small_sky_source_catalog/provenance_info.json diff --git a/tests/data/indexed_files/csv_list_double_1_of_2.txt b/tests/data/indexed_files/csv_list_double_1_of_2.txt new file mode 100644 index 00000000..a30f60be --- /dev/null +++ b/tests/data/indexed_files/csv_list_double_1_of_2.txt @@ -0,0 +1,3 @@ +tests/data/small_sky_parts/catalog_00_of_05.csv +tests/data/small_sky_parts/catalog_01_of_05.csv + diff --git a/tests/data/indexed_files/csv_list_double_2_of_2.txt b/tests/data/indexed_files/csv_list_double_2_of_2.txt new file mode 100644 index 00000000..bb12c6db --- /dev/null +++ b/tests/data/indexed_files/csv_list_double_2_of_2.txt @@ -0,0 +1,3 @@ +tests/data/small_sky_parts/catalog_02_of_05.csv +tests/data/small_sky_parts/catalog_03_of_05.csv +tests/data/small_sky_parts/catalog_04_of_05.csv \ No newline at end of file diff --git a/tests/data/indexed_files/csv_list_single.txt b/tests/data/indexed_files/csv_list_single.txt new file mode 100644 index 00000000..0d98af84 --- /dev/null +++ b/tests/data/indexed_files/csv_list_single.txt @@ -0,0 +1,6 @@ +tests/data/small_sky_parts/catalog_00_of_05.csv +tests/data/small_sky_parts/catalog_01_of_05.csv +tests/data/small_sky_parts/catalog_02_of_05.csv +tests/data/small_sky_parts/catalog_03_of_05.csv +tests/data/small_sky_parts/catalog_04_of_05.csv + diff --git a/tests/data/indexed_files/parquet_list_single.txt b/tests/data/indexed_files/parquet_list_single.txt new file mode 100644 index 00000000..77f8c852 --- /dev/null +++ b/tests/data/indexed_files/parquet_list_single.txt @@ -0,0 +1,5 @@ +tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_0_0.parquet +tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_1_0.parquet +tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_2_0.parquet +tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_3_0.parquet +tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_4_0.parquet diff --git a/tests/hipscat_import/data/margin_pairs/negative_pairs.csv b/tests/data/margin_pairs/negative_pairs.csv similarity index 100% rename from tests/hipscat_import/data/margin_pairs/negative_pairs.csv rename to tests/data/margin_pairs/negative_pairs.csv diff --git a/tests/hipscat_import/data/margin_pairs/small_sky_source_pairs.csv b/tests/data/margin_pairs/small_sky_source_pairs.csv similarity index 100% rename from tests/hipscat_import/data/margin_pairs/small_sky_source_pairs.csv rename to tests/data/margin_pairs/small_sky_source_pairs.csv diff --git a/tests/hipscat_import/data/mixed_schema/input_01.csv b/tests/data/mixed_schema/input_01.csv similarity index 100% rename from tests/hipscat_import/data/mixed_schema/input_01.csv rename to tests/data/mixed_schema/input_01.csv diff --git a/tests/hipscat_import/data/mixed_schema/input_02.csv b/tests/data/mixed_schema/input_02.csv similarity index 100% rename from tests/hipscat_import/data/mixed_schema/input_02.csv rename to tests/data/mixed_schema/input_02.csv diff --git a/tests/hipscat_import/data/mixed_schema/schema.parquet b/tests/data/mixed_schema/schema.parquet similarity index 100% rename from tests/hipscat_import/data/mixed_schema/schema.parquet rename to tests/data/mixed_schema/schema.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_0_0.parquet b/tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_0_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_0_0.parquet rename to tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_0_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_1_0.parquet b/tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_1_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_1_0.parquet rename to tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_1_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_2_0.parquet b/tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_2_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_2_0.parquet rename to tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_2_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_3_0.parquet b/tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_3_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_3_0.parquet rename to tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_3_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_4_0.parquet b/tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_4_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_0/dir_0/pixel_11/shard_4_0.parquet rename to tests/data/parquet_shards/order_0/dir_0/pixel_11/shard_4_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_0_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_0_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_0_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_0_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_1_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_1_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_1_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_1_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_2_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_2_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_2_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_2_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_3_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_3_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_3_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_3_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_4_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_4_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_44/shard_4_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_44/shard_4_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_0_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_0_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_0_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_0_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_1_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_1_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_1_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_1_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_2_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_2_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_2_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_2_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_3_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_3_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_3_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_3_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_4_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_4_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_45/shard_4_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_45/shard_4_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_0_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_0_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_0_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_0_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_1_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_1_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_1_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_1_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_2_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_2_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_2_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_2_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_3_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_3_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_3_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_3_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_4_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_4_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_46/shard_4_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_46/shard_4_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_0_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_0_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_0_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_0_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_1_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_1_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_1_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_1_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_2_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_2_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_2_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_2_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_3_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_3_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_3_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_3_0.parquet diff --git a/tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_4_0.parquet b/tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_4_0.parquet similarity index 100% rename from tests/hipscat_import/data/parquet_shards/order_1/dir_0/pixel_47/shard_4_0.parquet rename to tests/data/parquet_shards/order_1/dir_0/pixel_47/shard_4_0.parquet diff --git a/tests/hipscat_import/data/resume/Norder=0/Dir=0/Npix=11.parquet b/tests/data/resume/Norder=0/Dir=0/Npix=11.parquet similarity index 100% rename from tests/hipscat_import/data/resume/Norder=0/Dir=0/Npix=11.parquet rename to tests/data/resume/Norder=0/Dir=0/Npix=11.parquet diff --git a/tests/hipscat_import/data/resume/Norder=1/Dir=0/Npix=44.parquet b/tests/data/resume/Norder=1/Dir=0/Npix=44.parquet similarity index 100% rename from tests/hipscat_import/data/resume/Norder=1/Dir=0/Npix=44.parquet rename to tests/data/resume/Norder=1/Dir=0/Npix=44.parquet diff --git a/tests/hipscat_import/data/resume/Norder=1/Dir=0/Npix=45.parquet b/tests/data/resume/Norder=1/Dir=0/Npix=45.parquet similarity index 100% rename from tests/hipscat_import/data/resume/Norder=1/Dir=0/Npix=45.parquet rename to tests/data/resume/Norder=1/Dir=0/Npix=45.parquet diff --git a/tests/hipscat_import/data/resume/Norder=1/Dir=0/Npix=46.parquet b/tests/data/resume/Norder=1/Dir=0/Npix=46.parquet similarity index 100% rename from tests/hipscat_import/data/resume/Norder=1/Dir=0/Npix=46.parquet rename to tests/data/resume/Norder=1/Dir=0/Npix=46.parquet diff --git a/tests/hipscat_import/data/resume/Norder=1/Dir=0/Npix=47.parquet b/tests/data/resume/Norder=1/Dir=0/Npix=47.parquet similarity index 100% rename from tests/hipscat_import/data/resume/Norder=1/Dir=0/Npix=47.parquet rename to tests/data/resume/Norder=1/Dir=0/Npix=47.parquet diff --git a/tests/hipscat_import/data/resume/intermediate/mapping_histogram.binary b/tests/data/resume/intermediate/mapping_histogram.binary similarity index 100% rename from tests/hipscat_import/data/resume/intermediate/mapping_histogram.binary rename to tests/data/resume/intermediate/mapping_histogram.binary diff --git a/tests/hipscat_import/data/small_sky/catalog.csv b/tests/data/small_sky/catalog.csv similarity index 100% rename from tests/hipscat_import/data/small_sky/catalog.csv rename to tests/data/small_sky/catalog.csv diff --git a/tests/data/small_sky_object_catalog/dataset/Norder=0/Dir=0/Npix=11.parquet b/tests/data/small_sky_object_catalog/dataset/Norder=0/Dir=0/Npix=11.parquet new file mode 100644 index 0000000000000000000000000000000000000000..af44a7610433db34d1fbcb4f0bbeb0ec6f2d01cb GIT binary patch literal 8913 zcmc&a30PBC){hV{5zrte21QMyMAVm;uu8-wFM&WJi+~V<3ML_tND{U{h)PRsUF!U< zzwJ=Bs&&*l)}^D?RvmGvtyNL0wQ6-_rY@*|{jJqjZL9yeFCh>D+WAKNKRxfBd(OG{ zEce`V?~9Hnvm)7a_BIxK7bBHDo5sd^VORr$afpWb`Ovw(e%E&}~0fWROg)(zT+Akz=%aey}i{tDn@0FML+0l)#i3KC=ALVF5e1dO{5GA98) z0&otX1waTg4}cyE@EG7c(1KvTM!^3U@Us9BFqQ@FC7|DgwimPqfFA(x2w)VzAfWdK z*bB0u(ElDF9Ka7~{s3P9{by)nU@RNj&!Oc5e;>d%&_4n21AtFZ1ok!15&)Rc{}tqd z0sjo(DD-*IKM44bfL{an65tnr!vMblTm|SEl!9@f9SU#^^!5Pxy8ySKPlNII0UrR+ z9r~jIq5%ki(-4b)LOT#-B+!Nd+=KA~knIC_Jiya5avVkrTh7JV=%i11P1l!$qMcj z(woic!wv1*FRcH7fxK7vg9e8ShC~bq@8A~uaPo5r~x!kHN;Xl4i)|Bj2HIBX}3O~c^D zf(FCT_UgvIBhhqHYNkx4N-p>#G*%3Gr5pH46FfQE^31# z;9Uy-a-0DNb#wsZl;ki{(h{r#C*jIf#GKXTeR7<%Gr&xs)WsWx$C}nr8uEg;lB;_%D zGkYs4cyE0qp8q2@esJ2^Y)#YKQ{Ls6zdmf4++5d8_D<%`TRURd)rR>7X~T8N>Mv?# zAAFiM{Me59!}n#x9uSuwn9%!J&)Bpx{$&%-NUO%lch-F;*>Yn9LEpV)oVo{6CD`3N zjNQF8Eh@FnwAj*}bI)IJHdt^Z2=c<))Gi$8G%p(c*pF$c>^v$jnju*rFQ#NR`asd; z!Fo53XR@%>5MI=2%z)gO63oKVnJju^DkOmvf-eo`Vrfh+y>T3fsSuIM3TaRTU8)3r z3KK>q15W`U2ao|lN`hcxa6HriX|NbeWTJT(!BUu!3!3UjgNFK%AS|Ke=3+@;2b#DZ zYGpmWek_wmuZIdecvK_he|^4NJaY%fg9?+hUtJ8e^8)Fa+(WNpjZ+4b zYa+h!eMInK_1>bJwP1D@^@9&sq8N>r)~Iunxzcj#h=nE?A<- zx^^UIslxKkh3edrl?9H1lIk_hre!y7P9Csq+R%-ghDV@nDY)OSZFPN%40`AC{++>j z;wM-XD)@=ry1X8?cHu#%%gbW-PGvifR7MyTDR@DoIPAXOy8u|OI05F?;z5N@K!j@( z_{^qZg2w~e>K3C?TWF}PEUR?+;lnxNBe>dMu%mygA0Bgcq2d3>k9qtyKbWu--j3Yo z2@doAV@DY5z}5w!2SPZ!7b6afpd$vCdIRDa@JoZk*fdu~@J}zREHb#l{VHd+kKoB5 z!MQY0dq`XKd=pJBi{A0!tnv{w2m~$R$Rzjhb`w3{v;a_s z9_!P_}h4Q>V5qVzHO8QrR<|dulG@ zemDA-P{F|Wo+jW2ueZpyW{<`l<~?&728_i&yva@6u`V4?A9yE4bUXu3Pb5?^d$MtE zZbjhv2^!q@==$MvnkV6h3OC(|3z&@0y;bDhB@uezQ_OC*z3WglXvhd1s9q79^A*H^R_-3_SyryWI12I_bEddFi`r-zV1On`{BaI z4-bijvz}UKiCbcYe(NOaG4e6Oql+J`pMQUxP}?we&CQ+`p>E;QwYLUag&$OWf8n*q zR$)fQ!ui97RS36jC+lx1s)Q*U{_+>YxaqRpZS}$lE05h7teYvkGP-PJpZ&9i zg==zR;u_}&r;VOHX-(n+;pg){P1v+|2}JIPlmg!k!jOt%VKoVxg(o-ge%L*}Q8;$& z(_c@XY!vnkj21t|4hf%rJnz@#oF?JXF!CX5SCdfrUBAllC%zWG+4J)9QNg!`vr5LU zntu0=@NJvECY3q><2cn6&^sqhM65@#6+IK~>mM0K3nUWQYA0`mX zSO4wH4VFZreB7Z=R;)@S?pwlHO>ZfQPgUK2%Q%)!jC-~FifzUW;^-2^#LVa{g1iwK zxf0h9n!9~w9jMh1gC_HY-7_ZvDZTsO#LhwQ@g^1QB;K5upTBI*$HXVEeKf;R z`3bS~{e~wY!wwQr%ZqdJS05*A(G(88G_CTGX-^##ygqcy#KkSSVP7n@LVAwcxwdd=I#a{@)i6FdQSv`hAiq+pn=ilBO{|jz)!x9`3uOPVF^B#H@>bc*{IQR1>f|@y6!i zqF1h8f9I9OLeY<>Z)0otVv)wSd|XHA5>7*3LKwR{LMpXR1YMJyJojzG*=;+t;nN33xX#9KPU_|m2K=JnltPD7 zco>kRZW47UbfC3&6oW6Z(6G=IFZUDeCExU4_Eqx_xd#gVkV~52h1{?NFYrnvc#(IQ zKWamBE=pnJ2pF98(Y-M&6zBSvYpq4vD(+S0biT=2WT-9Fmzk?A)++u;-b8-8%EF(F z^Y~6)Ath*_M5gntT8jaMt<`2TNEDk4=AuGRno~ug%~6g7`PEjFzO2Z=2We}yrQG3W zlXl%{|g)p99SJ!5Yrxkot5gPAdg^Sj~3X|0q z7vpV%o7~O>p z&XN)7anhfq2z@?zYigywP*v^gf!8+c!0e#d?TTk8-Q4 z5qhSPDI`f|0-$FcNyd<}5>i2uC8WHBqRC_)d^;j0x%o0X$f7hnxAJh>mABCrNRdD#2R(#95lANNL^c%Dkn|(7PDEGoK>u| zC~W!JAeS5)sY|w_dF&b(ugOa3gU6kVAv&E8S);PL~iJRBiI$Cte z663St(!pL^KCB`5B-Xfdq*jtDN4eSFe?@VAp30$%R#I{B$}PFdp~;n)^;T7l-bC5m znQi4d>$BoyZnvJE`K7bwR_Rh8k5WAHrWEq=d3Jfpbhd^bxnj+==~K+@_eOiWe^( z(pbAT&uM!nd6a=_UYC2`8JDj2nkWBvtyNx>(X3I|f=$Naw5g?C<#(|*IoGI5k(C=1 z@s5n_mzCp@MW|kBqE%+7t!OXBQ8}M%h8m=B7|}WONsho;E*bGp2?~ZIv%~KLr zgh)gkm*@y3$?DdArjj~IlPM~NQW!N~5|u+#zXEiMD5DZfJLu0r0`2rCqJCDg*+HhK z!XT&XiQ@r9`d#|qutpu~jb3mM0RLla^iY%a2q#G}sRYeSnI=b+q(qn`9wBn-5+G~6 zA0Q@rP&{SrYY`&Gz#tSal5|=scJ;f8A7vWkL2J#UhU!}FN7og2Dq8!QX_OOAYMVcB zB>~If46!GwUHar?y4(K5&h1C4We$yV8mIpyPXB=+N5KF+Af$mTB2faNtZ7x+l|NMI zv}z<)Lc-#}cs)$k`Zz7exV&<0Wkt2YCe)Xg3#p4A&vmimIj?-7E8ySIy84S9f}fPe I!+&P^KZTnn_W%F@ literal 0 HcmV?d00001 diff --git a/tests/data/small_sky_object_catalog/dataset/_common_metadata b/tests/data/small_sky_object_catalog/dataset/_common_metadata new file mode 100644 index 0000000000000000000000000000000000000000..b473e52cc3054470eb114dcbc9a4edfffadee632 GIT binary patch literal 3983 zcmcIn+iu%N5EYUjNZJ4e5;%~7JQxJ>mP8jziIt}=DUqUT$+nV+B8x)cO%kmoMN?Ng z@Kb-I5B-3ARzIXOONl0Raa|C4S{L-|;QV>}|w2=~49qA*15i#=DhofX%>@6SPnJ7mM%m4{CB7X(ikNvnv@9Cz9U< zgq6OHw3hx*sANCbDzgpc25{_3I+U?^(?eF_fvr!+0qX+PF#Xf?0 z*4(IfriRya&K%y5X=SV_;>46_se3J#WtyMJLgUgXE(?ZI?(0feSqy_G?P+`5=ZS}1>T`--{C>Sn<{o7P_ z76aI?V08I%_qbg)CES-dMzL|FDXn~@FHlPoVUPAuz8nwiut5#CHP5g!hOY@S@^W>hGttD#OjQaL?I1oQhUf7^!)*`ELM3l^`L%9l zRn--rZN3nreCzVU!B`kdC$B2)p(WMg81+9bkoPT?*BC~=X}+c`N8RE&ro=Jb8r#Q| zL^v<;IvE$uUC!0+MX?q60P!y@>`s!WHZh9gf8*k|sT%H!Nvzv-j; zGX$UVYMdj>%Bg_4ne<=mYu!d@9Lp>YTe&4PLbX$&rq`I84&%Lxn{Rk`#fc`)Go4?C z*O?e~$fJ5HZ$^-h-{ZAqv+E2~x#D%EW}PN$Bgy+-@%rpmLa6I-Up}SuyK>v>AZ96A zzn$Bbg0ULhKetD1e|0X~HFm4pb!LU{#3rr3Po0g*3%wh0-;qaW*z@+*d=!`cwU*Z3 z>#VxPAyt(na2ocTm!p08-Pfhg(5UmiEuMsDpZ>2LFId>G)Z+$)-io#Kp5?qmp$Cbf zjWr7W+g8NTbSqGe(5pQS>aOJ*#S-h&&ClknDU=qn$NKK=nYx|OEYi7wdkS)RdMVGE zPNRa?5g{sk_?2xF#v}L$3%tt=yw+;$+LCde2MuZ9ea264c#dD}(i3XAc`gSNo(!4L z#BX-ru_-ylKO6BT{qqmG9J?;@3MFK=o?po9CY6|!4cUn8S7SNE=2w83W2CC14g4=? zkif6vye-j?v`%4B>Iz}JplYK&rRsbCXNXKN(0=NQ4>&7tHLHH zI1t!N0Ka}eKqWpHPd?d;P*{RR7%xI1ruo%;U;G#~$b)syV@r)Se|%km=W0E_Xfh`v z>8(HTN&?K`4p~%{6`nv^G5_jr{s_&7YP3z{{~+=o6aoeVaKK1~EFzddm^EwAeg3e} z=?a<+2pkTqH(|5&6EV+Zk97rUeAmGAJcSDDPd-^`!CWHRUQ GulT=w`r1?g literal 0 HcmV?d00001 diff --git a/tests/data/small_sky_object_catalog/dataset/_metadata b/tests/data/small_sky_object_catalog/dataset/_metadata new file mode 100644 index 0000000000000000000000000000000000000000..b216cda60ee09a2c05c172df60d8addacc916663 GIT binary patch literal 5175 zcmcgwO>7&-6<*1zz&I|DMj=5VN-AvXjg#2sk3`!{kif&GNG&T-5^Zxy7KOUIT#_s9 zE-C(qS{p8k0yT=9iX4gpHIkfq3Ucb92P5zy$T6oNhoXlZf*|L>$RUR$@6G;j`6F3{ zoUFv<%zJO<``(*3Z${cCg;;QI@q+{tp6@CR)sTYEGQqj?v*B5-$^_41qa)dkGgVcw zb%Bnwt9ClAj@^9<6mEYOM$G41ovPY7AKLg}A@D52*j8uKTIc){h^~90xqMqY9Fsl0 z+fXGP~dS`?Dy4P=1k*@dVWX)*Uskg zQ6TjFFSx%ga^D7B9)C(cba?z>=b^`AM0y$?-@Q0o6fbieGBcr1X1Fh3;QsxBO9lgN z2Lf6inGxw}$h^%C$qe(>&*srG_wn-2#O8m$SqjcNnd0mko5~+U_hz{7U*bN0Y3T}c zVGa``Oy8&R{ocz%-*GqEP=B8Zy*tDGgX6wFk62#phpTWJ%m=RwG3j>qgMNB}d-nnb zeHuN4Cme8FrviOz2j_v`n5c;g}_ z-K2C1pU{7=Lw(@_p?|GB$ACUO-$cT{FW?DniS8nS&;Q(9G(1#7c!v@K83lJ``l zhjK&hNmxI@{0^&{V6K^U9DD0#=F&Cx?%g4oQ5LLL#qlF79N8EP&Fo!So#fz&eH75O zyBC3p;}+YICP8f37)VWTNmF5bisOKw1WqD3!Y2&no{y7we$5!D7p>8S^u&+x%|3xA z5xv71RZkGoK8h#thVn76K6r8s?MMCd#K-vu)oK#S5l(@bmW+sF%Z~%X8^t7&D}AC+ z{(3O6&Bl~tz@96kq4dP#9?~7&QKeq1L#u$U$sMU1r(#rLn7* zqnD#BY*aU8e9HJ`_+jrd;d7)=*b-m2FOK2&7b5uFChLTd9r)n$TS8Vzs!lS5)Cpgw zvQ%nBKS1G{r%w&Q7W?qfS*TthWI$jXeB&hPD73Lp$kqUmUCtJ2vXSXkZh&1j5tFk6oM%vm@#Rv!R$bQzRLAFHvZ2HL1zpev zb(k+Emr{w+N+F?SabBXy@!Wnz%tvi~Os{4$Fh`*w?6vBumD6P-e^AMSKgfyC6^m=- z-WH!r)`}zCZ==1q($zd(e6C&L1>|K&VzFDvB>QDCrlTFir%Cb5R6{lj4JDS0s-={x z>sQldhtDKmh*31Zx1(V!WU&}6i-krtyXM8Hc4L6NhgfE=Vf$8TSH#0+DelIUXo|5$ z=TWj5Sg*`&e_SfY;X%2y6VKZ|K>T9^wdm(r>B{lU|Hs8B=w<7un0UHr>uAW?WaGHF zjPdPNU=6{iXxYn=!{i2!x#{;mQ>&D6mb@a+I85XgpR>xvq^_8`14W~}r*Uh`=5cY- zed{@zU$R;3%IlCv>mzy7gnWD)uL+xJYdDfCX0fNN>;Apr=Y67h-87OG>}%_&d>YYD z%WboWn5*{q$=t4r7_0ipbGun+JL@vJ$A-FTYZiMEq;+?=rbf|H z;`)J%bMk_9-)Nv-Tsb1Ptq)v(k5-u`POI- z(f&SQMkuLta}56$8u;+j*e_&ti{v+8knMVU8=8pk;K5;y9s0%>+ylV>#DM}eS;00T zK&g)NQffR(*3vM^HEi(o5|DE52dKmc#v Ps9)wa_&awS{$=7>40~{E?A*kgc>*KDuhRmG*#u$$CowVXs6E9hf9miAil|mHhk8n0?FO zjHzMKwv}$AXoi`09*-{!vJ-`u<3XU*JLP%B>R8Q|+08P)FI2fy{V*MW zGjtxt<8a!L_txYkd7eD0c~T~-e16dWl=bMaGYup2*1NwGlb1c-U+Y{gZzMUr*IAJ+ zvtk|M^nz&|40{cEu~XQxTZOf(r3?WC5I_I{1Q0;r|1V%~C!=sQe5a1i!(rSx9|T9v zSsknEG)wYlRVLR9t@Km5x;b@9HoIT-{`qY8`{(L@{M2{<{O07>pH1FEXSd~@y&s>; zowzdfO|L5N%$KLssokG1&(t@)#@*4!p}CK5@w{-GFCQP&Ly{-=y41P6p-!JrrU_qUn1x@2xry_h$r#_`oSs6LM)d3tfLzx9Dz_Tv6%a=FSoJvn>x z!7TftJo7MUcWBP{=zY)`*k?mG?^wy`L;9o_X_1uX(QV6KN0VMK35ORZZx}>zU>@Z* z^TswuE%Uj~)9h9ivpapSUn`xKcjhtCE;W-%)~fEvBt}<}$?FD#>cE?M!EBK}lzjF*x!UZvqr6Lt7C#Q= z$E9Z;&%M{nPxkvU(ZzD{SgHMeDcgK;ty%WNNo1eU8^0g@G%re3eH!Wq_3A`jCxxEd zQ}ezCoht6-{!Y)%-u`0tH%NAgQ{k{Tjk@nI&F5kNGZA*fc+)S;=K9HevcD;rx1UEa zUDdwW-J?{ z+HG@Evr8+-bl3RC-1)EB_wB_uR^L;bnnSFqXB^_a-Mf2D$+<||bPV6Tj7ckN?fd$e z%-Oa+SAXx;?%Vfe`o{Lnx9fb4ZSvf8_4-5aQS#kfw=eTe?ICOHS@rj_2Rq(0XKVJI zlYC$6X1aZ;-S(MntMO&oF?ZbUOO~|maivaLTVwlfjf~mW_mZ{J+8VpQo>R~CWgT+N z-OnL&cg>Q0)Z1-2QoF6j&b6)kr<}I)_IUsHF~9%3S=mdyZF??l>*Rdv*RPCAYxCTG d+kEP_3t+n4hgk#=KmY**5I_I{1Q6Im;4hr`cCP>c literal 0 HcmV?d00001 diff --git a/tests/data/small_sky_object_catalog/properties b/tests/data/small_sky_object_catalog/properties new file mode 100644 index 00000000..0a7630fc --- /dev/null +++ b/tests/data/small_sky_object_catalog/properties @@ -0,0 +1,14 @@ +#HATS catalog +obs_collection=small_sky_object_catalog +dataproduct_type=object +hats_nrows=131 +hats_col_ra=ra +hats_col_dec=dec +hats_max_rows=1000000 +hats_order=0 +moc_sky_fraction=0.08333 +hats_builder=hats-import v0.3.6.dev26+g40366b4 +hats_creation_date=2024-10-11T15\:02UTC +hats_estsize=74 +hats_release_date=2024-09-18 +hats_version=v0.1 diff --git a/tests/hipscat_import/data/small_sky_parts/catalog_00_of_05.csv b/tests/data/small_sky_parts/catalog_00_of_05.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_parts/catalog_00_of_05.csv rename to tests/data/small_sky_parts/catalog_00_of_05.csv diff --git a/tests/hipscat_import/data/small_sky_parts/catalog_01_of_05.csv b/tests/data/small_sky_parts/catalog_01_of_05.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_parts/catalog_01_of_05.csv rename to tests/data/small_sky_parts/catalog_01_of_05.csv diff --git a/tests/hipscat_import/data/small_sky_parts/catalog_02_of_05.csv b/tests/data/small_sky_parts/catalog_02_of_05.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_parts/catalog_02_of_05.csv rename to tests/data/small_sky_parts/catalog_02_of_05.csv diff --git a/tests/hipscat_import/data/small_sky_parts/catalog_03_of_05.csv b/tests/data/small_sky_parts/catalog_03_of_05.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_parts/catalog_03_of_05.csv rename to tests/data/small_sky_parts/catalog_03_of_05.csv diff --git a/tests/hipscat_import/data/small_sky_parts/catalog_04_of_05.csv b/tests/data/small_sky_parts/catalog_04_of_05.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_parts/catalog_04_of_05.csv rename to tests/data/small_sky_parts/catalog_04_of_05.csv diff --git a/tests/hipscat_import/data/small_sky_parts/catalog_10_of_05.csv b/tests/data/small_sky_parts/catalog_10_of_05.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_parts/catalog_10_of_05.csv rename to tests/data/small_sky_parts/catalog_10_of_05.csv diff --git a/tests/hipscat_import/data/small_sky_source/small_sky_source.csv b/tests/data/small_sky_source/small_sky_source.csv similarity index 100% rename from tests/hipscat_import/data/small_sky_source/small_sky_source.csv rename to tests/data/small_sky_source/small_sky_source.csv diff --git a/tests/data/small_sky_source_catalog/dataset/Norder=0/Dir=0/Npix=4.parquet b/tests/data/small_sky_source_catalog/dataset/Norder=0/Dir=0/Npix=4.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e52f7e2e7fc49f011a656d527997853d3728401f GIT binary patch literal 11523 zcmcgS3qaG=*Bc@Z1x0joAQ_=f5rsWK#Wuaa@f`9H)Qv68_F@dSfo$VdCYkwxuP-qb z&9t<9hmZ1+`B?cZ{g6pVz#m%q__9y!JNLH-V;g3D=Fhz6-gEAG-gECgcQ%g5 z<1zi2ZwD}c>y*jVFlnJQ+UuQy_J+}Ebh-BLhL#=N;2+)}L^kyT^qIcU*hT^^Ah=W*ET+@& zHa40k(%m8LU1e>9lV5hJdO}*?r7HzN(EC}Hr^n+GI59om+gOZgI>8plj zUJ#BOkTpR&Ie(3D(&cF{44$viuPEHmXt=hoGzcr^-0K{{^bbp*1%!mL1I;DYGPQP! zUc>ZbX;>>5tXrKq%ltWvzhg((P*_bG(~kyC2ml(*zPcL%+E(}0f77c+6?PSVVnr>i z<~>*UTak9wtDkSTSrNZs-P&^V0sheN zvt>5ORs3)*yBXm4jh4)_0Q)RG!Fqo$*pt$lbqe@uKf8J+1)yl_DM2m3Iqzjg)&u>S z-en70_JEyzzCRxe_!~*5RZ9U+KWrp_2l;C(%TII#*fm+!hX!zW$=+r@z<2h|D_RQh zxw)njYn5xv89mi!K5FXiRfU4M4#_Ilma7c_;tF41k|+yEv{9^mSPx*}?@q z;pYbR42SVOmg#FV@NXUcOQ(SVAAGENe>uR2(;9Oa%sa6+sJb5f(#rEbH$%LKjK8}% zfw-tT7zy4Tn7<*@)lY`B0&c&JMjfEBdN169aj|gU0kIXAmIx}w*gEPN7 z=U^8|J#UpA81jgzH+ScvjmM=To48~QAug%AwY8Be<*9_nLpKC%}wQ%pZ4J|@j z%)~8!+yHvv+w>hRmxZ?4ulI=F{Y7Y-K3~zz4EmV0I|J5U0qAw;IXT!2=UU+v_s0Iy z)VmbcmU>+shn_~OcFVn)Y|ieUkbB-bJFw&tbMM+)t97#$kSOxYg!IZ562(@}S#qn6 zL@RO(P51ASXs2b4-~Kx!62@h6=livy(ypHphl5+u?F|}befL&0cy&f-=n)bP7_$A& zI~z#!Ra$NF+z&})*v7Oien29@*t5O+Fk8{#0eeqRIz^(s-(6W3vy4P5Z!@$n948U? zZaHK09uj@`+Qogp-y+eU!mgWlV}U_Y7xSUr0J{WgVFCh)H*S?pAvL*jRK* zRoJj5l8=_x{=k|&Js$ObeOdjH1za@gy>$c5r}B`vQ`76G(&JE_WXr23=>oJvJ(f6f zG#))08uaUs!CZ8E=nvh4uEe66H--8&{bJFJmIdd@rCcOrBoOU!k< zYe_xgQBf@YVB(H=G`(WyF3mFF)6HeCE9RoO==*cRj65`p6VYu;JrC8dJUmQP9FKk* zy@LIRl8biee_Zp~7A}$)-#BsKkB7b|V|P?e;iEZW(V3r|=Al2vY_X2$!b623RalZj zfc&-=_3U>@fWi-l7xlUrkMbk^tIka0qX&vPkujxlXmda}<)PVpv{e%@iuYSATG%b& z@&$hZx;*LLxUTd0X!n%pNfRCjd?cMyBAeliEnabN>_1Jqqr=>jPN{P{=`O`^mJfua z^H$b@2|klf$P{KWgQ()YUkI*Kt@O^cGJ0pB9uKN;Q;i!PY`9U((QOs(aN1ycHJh#j z+Eb)!*U&ig1#X3O ziK&UwJ&e8Gc2drRs-{$D#md9aHr-aMReQrHY<4=Zu@+pvIfccNi< zTb=|uoN&}+b|-2RI8E{3gtw#)flvo6C@8Kzd?*goiUmmm(V9S_!t2Vq+ z6SD1!a!1&Vnvm6-1}-ZA_|i$@iz0x$&FW420KTK0Jaj+6=F$~R8^FbfZg)P6>CK;) z{{-+W{T+ELz|~)V8Z`ganvjKWO-y|Ypytzka~nh+Fv?fYba zJ*}NuUjV4651FI`c%gf*Q7Zwiw*;hb0{Ge3*K(TxzWi#L{|$hvdd#@Ztg8u;-C$3R z1o+eBDtdo_Eanc~Ab`=V4+2#He`M^~RR?hFE8BB-0gT$ct@1R$3-b;xXabnO_WFch z0PZDc)nT|v*z0-T?3xe`YhXBkcc1pE;nnM9%69kbnJ|s|et$e&T&I z0A9HB9DmTtESthlbh{+qPUn0>z=7edv;$LoC0`hsKw}4()rDH4k{J#^ ziwyL0`^kmI?CLfyQ`*OFuM;~0myFJyQb%J30TXo@4y0v9Km_bkUH6Yes_Q6sNNqb6 z;g=ea;5dK=vNAXIV!#h@b}{%A7#IO42JW;yj7MhVw)U56ARF<6ZzbJVEF=c3$hgs90wBctrJDbz$QHRZ)9 zlhUY#$tJ7805b~oT7zavdz@Xx6iZbpo)~2{>D46~Z4}HlS&gMtE`t4J#(1OFqSPoY zN|^p)l-2~(^(GzWoj4(N*vP19)7p$_Z$*^rXM8NCoiXi69yaMsmf`W<7P-)F24P31 z4F$U#PCDh5DLYhYCwZtWFq9~LDs>|~f?BBVJJjo-J!Zvl3wvvHARo6XrOt;cJMuBB z!jEciT~4frvcH$YUK!&mQM?tlBY9|bWn=qo<3c|yTO4wIC22d7hf23xY_HUf@Ca&s zCaFX3zmuf)SEYA++wdNqS|N-h@%;9Fy3h~v5^6Ou5S>~f`jC|zhz52?evOslk%lu-x1L9Z%PmQ~^NzO+hNR#sAheJsCFarW8qKKG#WdI$ji%-jj}&F*tYOE{Ke=p2!hCkzxW$V z5b=aqN2C&jj!4o`II*}Ly#gas+UU9I_A#`S6pAh(hzbJu0AeA@YLJP?1d%l|4SFLa zMuSz6CXLVMXP2wer52SbOP`|`t0acR5``?)tWGn;=T}P-BnGZol9W-T;#cxivV;Pq zETLGDUuaNQNrp>OEg&bMP$ip)d94~*r8zT+&?}4vgDNerKxIs|D8|FQv;>|itpdxb zkORLwKf6GaZm6JWNf|tq(E#%E3=(~X4&?Dv`Qn6pL0*D74a+6;Dt^XPg)EzEr}GS& zv{aChS12he(P^tP3@T%Gxgrhx!8RpjNTn%qOHNWoVu7@s?bUd`Okmc#>`Kb0Rtcn4 za=sM&%GLX{-NzhtS}MdYwc6cAhzmEn$Sy-=l3LX126rDzRi<2ic9I0gz@h-#fuAdP z#ilYbGYPMWGnRy2%}?a1joF5LK}Mn7Z$+s>R#}|Sf5;q@xKL%xD^&9mx!QcO0sAX4 zD$FW=>cea^s?rh*HOX?H@daPZs&t5By6?C_4i){SeXzFjD#~|X>+hl8d;Jy0%5r)B z-;1A0R$APVT`t;_i-z7U6FDkD2HY2tj%;g3_Z2sd46m84qg>|ABc1UROyW{)NBLfyP@&AX``vL|oM*Q8nQ4!klivQ$?BGlRq0?;>d2~` z@zi1AJ%uiWk%uZF|kzzIOMeuQS4NcdLz3ILi{N9Cy80Da)vm=Vv5PcbiF^ z0R1YJz<#!BjM6GK-%zg7<2pc6nOzlMky%9O@%iUwBVUo9QLU6ES}7St5^I(|zDi=^ z;<}3<(j9jwlu^mJo=VS6g4T@7s2E&!WmP~K1(g}l)Oeh%!(~+p9&@2XWMCm;GwzPV z4ONsS=f)Mp#lb*K6jDevZmD{V8WNcpLo%$!@32yq6@fhbnk(~)liFioJ zJoG^^37?xj>}pCvx!Ra#P~$#9fJr(mmog0pj*wKx#Knz-PD(OF3zRzRS3JO93H%ki z&QN?tK~R$%_9hXTf5l(ijvvlx7$;Y`_;XzRNxt>B=Z88F^Q#^F1VKfL(BV=YD-!2H ziRgch$C4v$XEb?PH3saZbP z3)eBkSiD|XzJxH1bMWV8QA+T`zkZT->@;L6u*&gLW9RTLq3nUKGyibV-Q$PfK!mI20O zj?cax)PTm0ELE0GwQ4QH)TO1vsE1jk@~ literal 0 HcmV?d00001 diff --git a/tests/data/small_sky_source_catalog/dataset/Norder=1/Dir=0/Npix=47.parquet b/tests/data/small_sky_source_catalog/dataset/Norder=1/Dir=0/Npix=47.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ce27adeda1857b9ba9b82e6c6f7e5a5a9096a537 GIT binary patch literal 137676 zcmb5Vd0fn28#jKHk%@+BH7ZF$Vlt&NNJf;AK1}-{j6G=>?MP)$noO&)XZeyPMpD_c zXNi!EJ$sf&+U(DJ+`m75e?0#@_v>ZeUFTfqI@fm2xz71~Z^_62XLFXh=AgcL%SJsv z^OD}?WJ09P)ib)|sz-WytnIdjY}QU5eh&KKXSg$djymAScsPDaP4RPTG=4hR8?vdM z!(-hYZ^&gWNMe(ZXHVG&r< z;AmObx*OU;X>0T1Yq1uzLb7*gjB;Rrz5OL{(3`T z5+k5rjEM61wv@z?usZbwPqFbo8x*vWpGt#a?`2LE%eSwg13l*mXb)3B$N5s0g*Ej4 z&Ch2o9{>kjXKd*vv>D3~E5#Q*H{=d5FPvWT>u3g_NA37ry25aYICW6Ff*AN{4PVAu zU;QZz@+DjI^xu5kpc{^pQQXAs}cFfKvKrD2Q|K9gb+qEINq(B_-*%3I7GXuzLfvjq|3_gRor9cWrj5$*FA-yS{BRl#cRoVyR6ve(r2`#+ ziE5<77mSx96dO4SG@Gx>dogL56taYjo*WhZRtm|LqgyWtlu?*U2RRzrjro%is!>b< z*;lgZ4BYdPpHEt}EmR6!hlEo{Qbt0hSgFVP1>}SIo+1rp z;IBeztyYF(v1^cH1a}K*nzEd%KF+`ra6x+r(?DFv>W*>i3uW|ysU!V*5rrW}3Ha9s`~}c?Y@rCK zp>u@}WEGVc=Hr~HBU9LA&Ty#`K7R(B zSgE2pkQptP(joY*5%wM6my-&yh=X1pav3$l_e&sq9C&>A8f*y-So#X`<<}y|1UQ{k zti8VbYpEL}!=}ll!I-3#Q%inOV1-;nZ3sq0VFmSs^EbmeyO0b)^=(L`DpK+jSmQ<+ zCXf>>5kqr-%v1qQ7IG?}Z3S)Nr_d>eO&u5swdeCFMy{e^=pVqeA(B#TT54)EW+TTN z^sbVN-Cu>R z%@(1pz}8Vgg_yjB38zj_vL1$ihKVLh0WIV^AWRw>trXH5MnNB7)HJ>p%e)HV6=0hz zrwFvU!@1em5u2dkIp7^Qt{YR1{wkz@1s!I}aKb}2zYTIKXa>^h33z*e*MXBtpO69- z7}K7UKsg9|4CaqV`&33ui42zvm{e>5ZAkMn6fAAj3+=k)-eM_Z(`R`V<-@pNasf_s zF0DiUT!U~CYKubgyf!*jFQ(<__=HiNIBLq}Bh;{XK3_qtd@dF6*>oSXPJ@sq7@V1z zFW1s*=<^ z5aos`zrc|-aP)LIiy(x>Po0!AR+oLW-TC%~Kuw|;S&I(ZJlq09ZNy4y^ zj~bLg?P^ytU zJX*(8P&MO#I;f0P;Pb?o!H6Fh+lmr09`d0XoVO2$#1F)G8zz@2wKPSZKr6xj2RZVS zQPZ({HT6P{N#*%CsOo4FWXD&k5MCbY9Uj#xHSiQxI6^*y$tQhyZXfhq#VE_xh6?lE!aOGY3i9RP;DFa2V!rEz`E*!WKzl8D z9U)jkd!R@X?p~{0=anI_8u(F4j<8N9SJ4OD*88M`q~DcVT*`PjWEv15E{Z1!b{H`|k*ClfOQ|6anF`#pYUwFIAE&r(0jW?y zh~VIIgtrPsAeW2~PBjJv!`WXM4Q)m`%z}WSSV$+C1TsLVLlM;pP*uQav0zgu$dYi^ zQ}W0V2HBvYhNGrc(7#PC!?Bx>NwB}eOR35P?0q=pFi^@b!x1c|Ym5}R#)H@DFu90! zAzl1AS{ecccR{I*koAI(dJzg8g#o>gXS4WixCYo_Yhu$$$jJpgw@`snL4{JRjh58s zQ#E-1<0K%GHi)csworETWy(QgqkQE_n0;mv~1L#VzR^lzGW&K1y2DA1Dqs|o(FmwO&i;47u!9NjG59!4 z=fbT{pj{B7IJhMmvoGUIXf@1mVOgb8zCI>9}D$~sb3*3mY& z{}tjm6!I z%TNZZSR&zm4YEy&U@36Nt)l)IITKl`3wB>68&!gooM1s3Bg0v!0*?R-qX0AHlQsBW zK=UYg=pj7xNM43oU4Q~1AImR=_0Zk0jC=}llJmo{P4lP>YTT5g#>V^x+1P~==pA2z(=CN26-sdvB{W!RmaLVYQ zu>(g<{VfG-3RMcpUCyN~@XC7RhC5uZU{KK`K@+j8YT=lra7ceR#+}i5IR!TrhkrK~ z7ryTDt3U|P$TF+9Bh@)It8o3 z7;82k8ob6}TSOw9Bg7)N#q$7BB`ob<$;M7d2ze@XfJSP$3`Lw8WrvocDr>0?a@W9{ zb{O+c$-@($lxlR7zl94s77A%&ArFG;C=d*h{5rZ&uc94r>3Fz)1;QMOhz+w8ccx%v z74<|cIEYUKB${yY=>>c_N~xx+@BAKP~?|VMhTV*LlQv!Sg75H$;ZG1m{U$G!PFb( z3Yh}>hO7>SYpy9#3&^XmEH#wDlu>^BFz3cTl_yIb6(Od&E5b>l5f1=6TN~&{esFroqS`;Cs#&(>o}+5i(3M<}wmu z2If{_TyG?pYh^e!!I*q_Fbi{5LDnL8u_O3>;IJs@Weo@IgI)&pY;r|nX2WrQuwxe= z*Iv-qlq%e`L^xl0^bvpWWQbekOhKXqU4erpN*TN8utP=o}6&|gxxDR5}Ir2z@`y$1=%aZ zvF%jQ4oo;5(OQoP3HasM_zS2K5$=L0>NfKSOgLVSwKxiENr7il8~c6=8Ux9?0!Nq5 z1X~Y*`67d(9MQVOFCZ^vD!rETkd|VcooX6^zPCuX4e~mQE6k_jLLS<10b%N>QJ#Q1 zY8~mpmqq+kGJ`VjkWv}2q`Oi8e-*H?Jg`~ZMG+7G8qA<^LxrDqJ0rf-Q zCyoF)ipBUZMjEnIvgs0HHw`ho#_+JyYiOTRLd)b*+6g@ap|b>TFGC2XBGvR^&snB` zM#GK_MnT`OI5ySe?G|C^g4}4v4s{hWs?b-<(UqfZ(4h?Zz{Pl5KZ=@kfh0d0kUzVq2shiJ}&j@9Tp3w=y0@l=ZF=?XtDrIa2) zpYcpM_VhB8({;E&>)wUAGzY=B%V==Yu(4|r4MIeVG42*3c^`hBtgBKJIJ!6AR5F#< z(wlz^uL?@5u`qc!)A4u>p;mBWVxfks5Kf@)0d}@TD6x$xBX52kxfbH(0#5887vYAa;jmn6*UA2E*0122 zH3-oWxbYNU_hPLf&q6$o$aNnHY_ZjpqX-mGfKpd8iLsgEhRiQRB~nM*PRNebyBeZKNjad_~sM+@P0DRn}4d@E~FP?pnt zgpA=pYh?imVZm*L)>3YZRmsL%trj<92OLo0us}kVl|;rEyA}EjLfZ6|*J1w^;vla@ z@q?`p3&)egMSUX1)>(ydRj5(g@I^#S5ppbA*p;euKwYo%qytSs`v}Nh3;jF8CSBEK z1Uu`nB+OvPIOy;OOUeV`)ZJZmpD4B?Wu`M)sw@=ZqM{-t^eTnN-7udeQ-QtS7PV72 z-hOnS7r2xrcc38fN6NLd4w!=U$cCO%3w85oQD{hTZ&85<_YL^cRgN>O5JfJQU=?gL z<`f_WxfD{VJ4l6wtli%8QfLm6R09io$%VL6^2iEL;o9paUX2P^$)<)0 zD0CWdd=`NYmEaXyhy)8|K{mz*F0Edml7h++hVu!@{DN7L>agpNEiwkC< znBtgntSmLn;LB)4A+|H$|0}9ozRvfN*P5k2{aiNN-#$%c+H?#JPcWcxk8{^ zA^hnh7vNqfz}BaM4*9f_!8?jlfGD>ir=+-!fJS?7MoR9O;{roiC#XN*h)BqJz>(oV zK~0W1-&K~=kNR3lXH+yCW4{&(kn?I}jRUR2dMw4dJRmPauG`WY%zF)EE6|=+Pjo`A zpphIk?f`7;2?A`*2nbW~Pmf6WBnot2==6#spj(KZ1#HzlH|zXY0sWHrS~|c~(Mn4} zM_hkuDHNMRA|~_#gM}sA2sdLXeO5|wu;aT@B^RY9ZpZNCaSm?de1sGk{}!pZ6k4?h z+Mul$`fePpQfhF4(cp+np{H`*zqiFkOR?@9T}BHS4V>48ZL}PVN<)N5&nG8}I$5_*S7Of1Cp7mDmhl#AtRnqHUyO;d2lRnYpv zRJ>HA&>2fE8&1e4FNENovVt;{0z4Pv;}|2wi&Y&JB5X&|rq9Ra9c!-uTHk}RZ{X@c ztmjo|b4Dn3ViF@LxE)HT!-!ev?*NN;%5@)^75K`ON^#0ENN<9@pmfTWBylKxWWmUFt!WQqB9J#gS&O%HL1rJRwe3`deOhrayP@JFgWWCY&Zmi znxKO<(?E+6(EXT4AM+296VgEUVynUPUK!n&i|8=Yb^>(&f&jlzieLwu7^Vyr9Evif z7SXGsq3Hi64@UyvnM9tCdxr!4V6-?S3AEkq*Ksj|%H> zb&`-hUxUX9HBP`b?7?-oQi;h$clrM<*P;j);%+6tBafJ7^Yal#T+<;XN-0CF#G?ji z=Eangi`;=M@%MD3#B@w{ACpGH6VdqB6-ulF zuMfP)FHFEeQA<7fggS;t{gk?Tt$K3~#-h?)u6ZA6|JxV6{?z^ScZ2(=*qD(YiSAJ7L%mnrs9B;kjGj($ilW0%4{JvQGA0?;_Gz1 z2;~LqZ%ormwXY{oC>E)3N8isoCBwRhCmN96gIkK;c+Mryzd`w0zCcJtN>F0#Yl+QelL* z0|m+f8S3iugQa-|gx#VS( z>@TitppCj`-2|v}AAb7_6KybjDq~ARu)KzP$#Mbpkk`^8WjJoVxSq=uIt8V8beF#Ema-qFU;G4-7=ITBzq{ch|hz0Dij|A z6tD$IxmqZw`;IE6XB?ypL*xp91cN%j%VGGpk1xh!uZCuF5>S`uF60R)q1E(S$-_>B zubT|6ftWl|j9Rg4!+M< zCg8BvWf~8qS~)(w>h82!V)4W2H!`muHm+Yxna532J<|Peq~w2SPUa_t=9h#6bpI_F z*WAF|z?#R74EkRukDXxrzrQTV;s4SKIoqt8tUI^EA0@r@jFO2<`T}BM?J9_^qK+Ce zs-n*Ln$6WS5$Kso^ehy5Je8iMM$fuR&!$1omaA_k(6^W9_fhCOsPr8*`U9%;of`B7 z+;%R4cCM0k?uvFoRlA{@b{Bp}mZ2FhgK4OJXobVGyn|h|m~BRv9d4Fi>ziL@A(ULjzwlCak(*r_UZnueWT#m;D8tGI?)0>d1MVV=TJtuoxK zG2B{ZxV^zJpWAVlpyM7%$GwV<1*(n*G#w9Bbv)eAQN!(2Bl zRj0EJoyxgJ7X(I^Bt};hMinZf>l&k*RYtcOjHr|aTXgYtY>ioH(a|5?aqo7NZq)UsUOPi|84^5X}RbBoxbRlD| zzB8BQ&ox-YWoL3b9_1R{;dcJYamUp2tEtG?%*)wqw7;3p zA~SKOneS0EzdL5*znV#m&G8==2Kk$ZEHam7nol}v9(Kok>Q{4_vBeB$i&_2_a~4^I zXIexYwTQf9vEZwP!nkXcbJrOEu5pXHCS-P9aIs%vYfl+_;<@&)J>h)ZS&D?Tkmw+{9ZE8e^*>XRBg=tCB@lWtmndk6N9+V|DhcRk^YC1!wC^{?=C(SyyCQUq5Pn z^N#hcuhvz@-S0YgulDa=v#7f^v-_i?-Jjg){_Jb_T4S4+&Ni?8ZQd@jsmrwaaMb40 z9h=WzZ5oVwG&=Wa^6$~Is7G>JW{)37d;Gf7BakrOeV=s}3y_bvq=m2}4D0^|1y>F4d-(CChjrJ0g z-T^MXg93VoMD>lXE+;egI<{~g1ZyTWvE)S$sJfrI0s2Pb3?UUF=3;=RF1 z-v+0c3Resgt_l>ci58}23)7AX)9(p0z6n*PL$U@9$q5{i7d=FsJ!JE-AzSYa+5T-v zzUk0igNE)29J)7pXhHVS1ILCQx;OOjx1k!-VMT+66$cJ0i5^y#J?!MMVW;m6JNs=| zxv9s6K^~U^J+4H1RAhTxKjv}sp2w|k9#y8p?+zMX9XPxudbl=w_@iUPpWGY%?A!2K zQ_q)!JYNTTzK!;*%l7Wl zYQI}eepP17U00?$h^dKTv^mV9V&+LT^Q?)fH5>oZb^Pm~@o!_s*X4}=P(1!q_4vD`DqKIv$r8-IsK3mT=Ae zP2Bv=g8eOG{du|mmdE|A@B7;{``ela*trGR2M6?t4RFW}a6BF`;C_HpbAZ4+(8Vp# zH8{{cHc*%wIP`d+$NfOh=0K5oke6G~=-?oq*dTFkkniyzzxzSsn}a0g!2xc;LBYWx zvBA>Z;7P}W!|n%9Z4Q>1hs6a`NS-@i8;X&^I|8eb0=;- zK5^^)iQAhe=9^F2O zlTY8De71RVxp~+Hx3EjWVOL_qDsscF9}l~EKkQa>Se5ydyKYmegQwKQPSNI0d31cr zllxPiHBYHEpZd~m>g(XCZ)2w>*X2(AaD3{g`%^zRPi-)t*622^DR^2-?6kJrX+Mrn z`*nZXpXO;~A=7u4u|i}9aWZzEtYeAHs7BVgMaH$5ZsI=OEM&Sx+;m>vbjyExVIMN1PuvWLycv!qGX~VmaB7($u$bxMKGQX1rhD8>VcyK4B{My0W_q^F z6j{vja-TIiWR_3dEOFi}-;!B=HM7RI%#v8l4sf3x6f!#`ZniXU_N0>8VKuX-w#=4U z%$eanXI99ZIdOBs^X5d9%!#a-v!G>;!eVZe``nn2xp8rG6Y}ORDVdvCGdHPaZi+?t z3it3;A>nJ{!c+6Y(@MhAYr->H!c`XYvfSt8gv`r}o2Sm3x4C5A)|z?STju3kMC@{p z*b@@5H!h+eFXBK+#G#sq!z~dSi}^+F^NT~~m&DC4%bS0)Wd7-z`Da_^ms><$aF4tc z5_u&qvLY|?dP(HXn#fx%kyRG*yYBMp5P40UT$?9k1qjgfSX4Hz676dD^6A1mD$JE=4_>_P0*)>v8BxEX`vW`)MhiH{547#C3*7x^G= zL2I0%Ykbt;_?Xc6xcK;ljqyuL;}ajmC$+|>bWK<>IAK+2!kYMm)Qt&gr3vW|5;9s7 zR9zQm4PKlRx;QU>v3ld;&83UCK3KfHb#Z>zCA$VM*%P{CZ~T&ijY|%cE;;mI$>G)| znyyQW1}`lRU0M>qv~1(jlch^fKUjLUb!mCm#0!HHFNG#viBGK9n0UQ3@#cfXTdj#z zU6XeRUDMs3q&TT1N-f|P+ zax>|2i-hGo^>WLy<<{EeHf_soc`NLME9|8!`XsDyP_J+-TQNYp!l`YAfVa{`xYAX+ z(mi3NP`z?!*-8)XO3${HBHk)5;i}QnRXz!;#OhVPWvl$OtH!sjlJHgs2v-M5SBE65 zma11zDq9_6$qSYr@rQBFff8YS%1iTchBujS{Ynk*Iz}%DrxGPgw#}ZYFb%px;8bVEmg%^mnB@6BVCu5uuiRBx4CTH zR_(g&ZR_%RX}g4Jd!%W56VeLQX$Q*E4r$X4x20)#>x+czi>2#J64sZg*Pkp~e_Ff# zY}@*BUit-L`Xy=lm4x&Pb^7(P^qboBTW#r8Jmp=XvRbOFNl(wu}behDPCrCh3Njgbi)#4L`~@{L*gt)3$-Ssq}}a zSQAwSi&gARs*WdAMh{h;zpJ?2GEIhLnoZ2KSe(h*lxcY))B0hi&G$^(ZdrCivg{{j z^;w+duqn&&MAm?ZSx(=x1l_V-hGe@=%ywU#E!>nn^hCDD!)(v**`jVaUPE$5Pt5UI zoFm?p<9i~UlgxdB6RgC^#NEY6i~%AI7WyJq5led_M$S(~%Exsd_48_zI$ zVQkdQ+0*9DGS{~#>20BUqi<1rN3YKwwO$|g|I5|0+W`C@naEt9@WTZV{eS<*rn3k2 z|NsBTrl!a3qx#wUbod!9s%o9rHSv42NXE%nS^FbeH0|=u3tK#!6BySf9nA6V?006j}p>&zd+F9|cF zi?pCkYh;sj_LQtRXc~t0S#zE)F#w+TYwr9P{z^aQbSAm@_hTMw;4t847 z`V}bo>9-*UcKF5|8Z!+3c=KV`@-e_2?6o<17@zG@aB&N;Yw6wrF5r(jze8OKv^~e! zw*y$`SJOBgSh=Hgw$pqTP7p-+X0cyCT>lIzzepVgK9Iq@f+q-g8mSHe>K*<&60B1Gp&iL((_2E0=6J zuLh=meDpLB{09#FN|K|0-`PbD;b;#}J+xyG^o!s1Ix-Hp&#Q&!4EY=8ha~Ifu^%>N z*hAnO&v$P7 zz61I;N97od0ZvVyxA8CVT)}~-+ktNJGbM9xMvHFRHt+0I6)iFo`@9bV?z9QGA-sdU zof-SZ2Y4chLub*hzr9YD4s?y~8FBkI*4gFX{bmA{n>L5_1I|`2W6uDpO&)Cb?^9f;+JdDqU&{rE{}jO_imqE2K**_XWbK+7uq~# z%k^l{Qo*m;t-w{aqswA2?qQweRx~g`-hR<4V7H&_(HY=Z_d9cUH88j#W8^JhSeHkq zrU88|d*{~Qz{1_;1-7fwJoZsTuO9JE@^)E|-9@XiYrH-)YyrEzp@X|#O8XBL@1IToBZn|a8qJ-ulGQ4@mlY>NT|eaxhn#Ir-E$b;(!zS&vI!7 z>c338z8g5^>6DxbB*q|duEkxT-_M_i{DHor{3)rxz^{#)bCIYVmkOgKV5gjJ$1;F* zdzWNS1nz&RwAZy?TVm}!3jCKEH`JX+LOr|Q&|C!iNZL!yX5jmR8^g1Kb!|g4P6KDW z$S9ryd^z!1lnDHz?b^o102SS42JC}g>`udy7eMablzkiWK|c)F&h7~M`Q?5KzhInG zd%xiSfPrIM*T(>BE{@dd7FdUuTY^)7qm5R-`+$V_YV>ih0vLFy#}!?oyz4O`(E@1r z;&%39%r~TNx%*X&SM*BEmjIJIhrgeV_FuQHpILxD+P|*TH^}1zy6o4-{H4u|O0NTNP1Af`3w`3||Fm-e zmgm+#C;-Nezj$OD(A0eU>v2GX;)3NHvc{H z0JdkXJANJQv3}N{eS!IBt&@Wx_qIpJ@}EG@r2N(}j8A&~uKXqFQYLBhB%piEs76=d z-X3ZzG5QC3^iJOiOg=yV%y{6%_+vB9qJRA8TqAGLwp1ec=4KKwByNv$`8M@FM7^HmOVcfzP?f#^*Won|Q3fRXF6__|d%D z5xCQ&#Fq`e5&Z^s9}aqCtHH1HK=r+{j?uu@;}+ z{edIc=WUvFd;{GETmhe1Uyr^EfureQYZCg`wA*lh4)SJp&56T%fcrYT?DGTqg$j?# zkZ;}hkMQ;eX7*Z}uZQ+=){~PbgT8a}n&HHQ*e?#P@QgYTEpiMGp6v~sxP7g94ceCu zdGRX`^z`3{yMEY@f-wJkupMa2W^Mmy;PEex-|UfZdn*p!8w0s>FD{iufgWCD;SvCJ z+_1sqCQ$TfnXC@@V3x)4hwyvEz3UT;fz&?o$7txW;HK}#NQ{4f!sWlCh{NodXCI7! zoTO;~|G?MYJigQm{h3{2DpMh6?3QlNT+rX;S^88v;FG|NJx74kkF;xZLw|BX$(l*P zQM;54h6Px^)1NN73jKq`M)TT051i1Weg`mY@YfYCXcv5#z)uBjeC5td3+OZV=O5v5 z%-6WG^4-9ZlMU8CMEmcYzLpPwue$##x&}NK{B_(b$XT)O;MCs8kA_5P zCx2kU-S)WbohPrN|IhCt#|gmg^%tLBgP*>N2lP$>cACeZ zrt4RIA0ciAHr)PcY=ZXf9G64|#(Skm_Y44vf86L!=r5|c_0S1)$G^MBUj=?Oa0$Bs zeO<RT8?^8DsTnCi zms01QdO**z*RMLmz8ej1Cg|34@o?^*QsA_qnY#$`vm>AF5W)}U+<|{Xfy_Q>^)le| zUdw~l0(-rCvd0b>J*3+8ImY!C%Yv2ySH1J@`~~fvO;Mva0WE?ij7WqYx2Hv{5&mYznxn)d82=eb^CXhDG51Zo+ z|Me~W?z#P(sw3!cv+vrS0mhGZ7aT-B#n1Ec>IMw*yTOVFI(?kds|5V15f5fP1}>QK zX$2qr&5tby=IA93qTCQ>IT|jD#5crAmH6(*z;%$9IVg}xL;kteO?VFc z`lZ1(EFJNk9GTk)IwYPNZ=@nWTu()G1pmh)-KR!^j%+g+#{%8cuj}M@pywQ~b8G}I zv8z5h4A@^geD{Aq|5^179LQNweD1_$@Go6DXXbmLvDt*FuYpg44DP)G7RL@b?gl90&P{<>f#By$JMCiGGG{QVM7s6H8Ek^%A?VuD+YW2h8#wo zX@=`ZABh%i95Cd-MGf+z++fBg&{sb-6#IZydo74Nt)ugnz8HmZ&o*|R@E_W9$B(j2 z0(zP!Wi3RzdQ{GdVBm1QvUBc*@W=26oo@q+vX*Q#$2>(3O%@#lT@jRSvIn>(>1Z!4 z@O>u@bq7uwI(%XYu-t2YfE@g1cXbY}2EJ?8SdfYK)i2elO3;-_9k;&$c41x9mH;gx zxqn9i3zd^iw&6TE{%cm%xkI=B6?OP^9N1OwiRUfglLpzP`#|3S){^t!i!trlt=$iL z@2UfvtJwgI z8zO#?=L-G~_76`Vz`WarjObhdtaxa4wHRpmvF5+CK-J!>TULWVGwZ_iYrvVixAlku z{>WaQ8;vbb{Bl1=w(G`5HIq_iOdN8{L34r%f+a1GB!F zpBba$``v$U8vM9f_EP)`c%W~fNg3K%?Hv_Oz?&~#uK%Q)H*&( z;P0ucCVWG??|@ilAmU;&sZ+29am)SmuuYc_%7Okvf`DiB)^0n5d55(vw<-bOsFt|z z0`Mixe<02Ry?pBNBzw?f+E;dN1pmDduk}}eZ-1QqCx&PeQ75Kb##_h3aKP*g-x(`0tu5_Uo{>okPVQ?{U!a2`= z7l4C@p7=8l_+sbARo8)Q;`bfa)s4X?0#BX=UVR+&%^CP^vDK_?kP|bySyl3CDn19uunbVfTZ?&yy%Lc=59ZzSMY=FIs2TDd-gZ}&R z=&R}IXWbILUIO}^$)6Fn829SYPVWcMoBQpD_62ZYMD?6`Xpe2N*!TlD-)hdC7GQ=e z*S{7hFbEv&3ckxPnh$0IXZHAb=FNRai|0irjFqQAU8HPQ5^}sb14C* z`U4McY)(7@%&mHU<0<;RuNL*F0zP`A{al3k`;IS~?E%`XlhqM3VD}exZw%1>o>yzy z1^6YSRD1~Sr@8!3I_#sV&wGveUpj?2^uoNiq<8)pLw?rkg-MIi-uk($>nU zuRecp$TjpIiD7Na1V3vB-_H~H#nJa_AM`I7S0}OpF7}@3F%ow!ldj&3Su)eXA;kZ0X15a9^=q& zKk?Dq4e;Z`_6b9E`M=@mxc9nqZ?)PWUzc~2*gsB0x4!`Vz1N%f5NI|0CG+$a=IdqcIQBN)D@K0~UUdWKbJN|>9@n9d z-;(~;6*$kEu09L6hJ9yg)m96jo8dyGKKi|?*9PbwG&h`E<2(zfxVnWk7Z~f$DaymR zRF5wm(r=<(X-M9-5pwgCuPXKeJtPZm3;MMf-8%-4nJzj|;QJG#s?k%@g00KwG1U>yp5?!+yG< z0QBt5iC3$kN6XoJF=4A`_?|G z1^?JZp~&*3gB+;s1YYw!2^UJcyz_ z5^_|?qlZmabAgZd9qZtRcHb|5Ki>r1cDvn=#0-2$Y+G1TtJHl^I3vFTnzbl#R|4oe zZNKRO@cn3)s=2^cM)iS(aRG*RmOR8fcKi7){XpM8H*0w#Xl_=|tqZ~TYEz`$QUfBEX#jUC4P0^hg6+j}g=yf%UvvZoUEdEFEP)jS$+@9q5mt3L4kvT_3`aQI{UJWewsG}m&|x?F%#|QCRCa~fIbmB z@;{tJ`|8nG_Ukb4s;BHF=IOEd?D{tJ-;P*%Z!)mhrZ#vw#x*##R89fk$K9MID}aX; zxy5|6`$bJXRS9})c8%|-^^mjLCovTGV^r~k6TnVcL2HxJkWXo4Z~TFK%7T14qkZ+4 zrpWCN0*xx=Yd;>H2KiFj@ z`W3sMcaXt&8Gz$qqy{rjT5^NtAj2)qv*iL%(f7nr-P|D}(3Z+O?Mi$O2EA3Pc~qgxp8 z;NGE&b?+Pdi=9em;(g-u=o#G(0Jq#2*=r!)MW%b}1vcW{L;p+tLNze!_^9=kXpiu( z&MwBdr^?hGM}XO!Ay@C9e_WW4!FOPR#%h=xd_A?y^xl}aQFY!W67L?zqc@c3-qGfs zdNC>vsP{Yac^&Lo+#>1E2XX0HZxLN0XtKNx7$s?%mq) zcgz1_@6Ert{Gz}AOoxHGGy+Gij+i# zk|{~4WQc^~e!f0y-Rry7eSh!&;QiBM?S1yyXPzvEgs~RrIWG-uK8aIw`kaN(^ z{cz>tPsl??$5*wz;8tc%rW7#5ztPMdestGhN8dd7BOoN075y$3{G2U`{@Q5;FeZQr zmz2h}(eF%FeOVtcHT%;v?LE_v`29k(ycBU0JDnhp9*`L>ME$$v?iwkGoAE8Zy%+rc zPLfC&0i%u06{Vn$O0{=60}4$uozn;9U#XeU{+_Y$(A)AA{E75mof-reN?O8W;n%Rk z*_IQsT^~aPtsS}D5j$8R-}v46Knh&`s#|Xisx`b%q>XEStm1jv`*4-LHp)ie7xIGr z27IvK(eq3w1YCdHkj?-fe8i{3lC8mXql22Wt9Tz$RaDOfS=K}3t1IjlZgzCiK4|&Y z-s-4@a;~5BxBI|z3PGb?E4bH?<$U5ed@1E@J$CQG-ppIl&YOHG!^OPpexUl=k>+U7 zQ*-#WC#Vv`eK%;MFJ#%CXO5xvcm=MVYV567t6~d)J8#zLdQk z8YhF;d?_d7s~*yUGZV*maOpsU3AV3b)gOfdd)NnhXqCsnkNtqc z-W_1rG2@ntVBo+NtIvpoj!G$g4%wshsbM8#xgTBU)F4x}&F-gwP4fLd&WLZ3^x~;@xXj6802y8 zRTc*itS}ljuJfgwDY+P^2>)4%tn32d>*vSzTA==^%(DyLAp7s#F6^*(srH$wNBKLM z0|rT8iH!6KK9oDJ{nc>+BrfjRz6Sj_ep`369C5rC91ez(Q*2Y`HXW?^G!-`ply{|-3c7n>oC_9LjGYbC&}@Oz^Nzyo}f z_TpgT>u3gD)H7iCq>&2tpJmAOp|#hPBOna_NpvTrV=xbg$Ss@QL0+Q(fkotJtCZod zHJGQ*+zVCYp=Z4 z3T_B$n-d2^k5zdLpx&(qrZ@J3mJeA=6u?o2Br$8$i)HV6eF-$QoLstuyiUanM%IG% zTV)Q@{IF;8tC>Px@p$8~4|EAz+a!tpV^Iz(4ZtVQWYg>c;J-A*k5!_>a&p8Xc60v1@0i2fd5%7kc z;Mb0$bl{!F_vfS=@jheQyY0W=$6DllYz1To=5-m*Azxghc$*D!OJ)!Md&EKEzxVDF znCsZl$psz^x|&Oq&j<23HNx)ru^ZK5AWQmUYA5{jSk;fz)Fa;A=6v?hPq5^l;sm+= z^ofSS|JJ6x+qPEY`Om^K;fMGv|40+wkk7Y#>WcuE+n4PR)Y9G;mj2=cQ@A8+3Sswi zYjbcbXzP?;R1E!ozl@R**e72Nv;6@2*Et@#jPj=HypTRn^pIXM9rU6^y@UandhbMQ zI4wTgEupgDH8xp39>jV4Q1qE-&{`#o;WqSb7Alnu@b9{{lPd~blmFc@1#}kvwa^DU zjd7Vp+BgIhF_@G=E>;N_?*RX?-4V?L`9FEozD2o}Lhu77$T2*+@h$M{_9brd1LTmq zWrKq7H+)G!@ipSKdm3K;1M;V>B`mYxPj2Z|Gw4~w+0Oq4^#un`g@LM;4@SJ;$D5il zp#!G)sPFs({&5)PWr1Jb-RnE=qrS`5ZfzS-Guk-)ILaxjb6*!Bm%Zq2StalsHJnuJ z2Pa3Zq$Mz~e9hMDI)P>P*v%BdWaq)@(->bW+wrV=aEA4YUMa>;m!bX9V^HIWQr=&X zPF8{=2jzPgdp-!j--~a$F(**J)<;h>8s*&M2U~`~^5a$kCdh|(^x9-UP}9~)L=@a} z`sRKoQ1o*~+&pM{U;L&x?1z)a_RYiZ!)67&9q_x>;`wkI=(>l)QU?6gZktmNyGL`g z{7c}0-G5`Hz;~Ii@;X4SlRj=|kdNt9uD1_CTivxGkC2A}$!+>VkP9aNx;8_e{`GE+ zGvqgK(;{i}ovDwyuMlyZ;(4py4Zd9W*y<-ZFvJ-!fOx`PY>bD|{_GDS@*0dswyCep zZp_R`okR4C6d-r<%uP9gdln9|ltX`M+lK~y z_$l3GVx|CDsN~0??T9Ot!@O}V;wojPTg-+1TV19j4obLhEia|{Dq=qFw~5#dxiYJ% zAs*zvE#}G#y>-zpxp`ayTzPF)w-I*RrW@R^qWoG%srXIk$KJAq zr9oDDYMYr0s$QS5JPZ9Mfq?h4_I3^=$VETlB>QSmbuHy99Ln5^3rPd(DvjMxyX%AgbCK16jMFx7P+t z&GOsG3LZ4$KG-XZ`SsVzxCVAX8Q~$ph$F*9Fx~_d9P<$l1wYRg6gMGGo0l#pvShFh zdS`qtgB9^N6sND_y@{cd^Z)!kxqv_PBnx#Ox)9WMO$cpKWE$=@F*HIu~{l- zh5t7S{6PmGQ+QX6X@5_($TA-Ey@`F2oAR{~%$|R)kqE|kuYF>73*+qatyBy$^W0e( z+TYn9cM9HSf!^xw-4$!-xk{N_X2GdL8@|!b{m-&q(L^8+m;*%veqPA zq}@Ag_6^#z)9R@)eT;?OekFOuERcCx%ETJ|N)!K^=mpYq3u}mgys=u*wC}%4H!P7? zK!=L2*Lp$6q$1vVlsi7j`YZ+6wI_6W4()jPM2nd~A6T)lQybhk+Z#26cw>YVY+|zT z9!cM2_~q3lmT<_T zas%pYU}L!M2Nlq3xPff}40%Ob--|dbF5Sy!gCC}nIXiyvn4iS{W6)oj=g6o+9QCSl zlA@3+p3BGVLHQQL3pKRwjegu+AU}bg-Ek(HcK@2&@#Ec5(0BIC;4I1?=|_ddf$aB* zliAQ)M8E5_1C1TTV|K&u))yLYc7iXzeDOLCK1|s~)derqd34gg=c2w>_`Me-Lgnt! z?!}LvEAi0o)0Y*BA`Cf-zm@>Le)dNI_qkRr?% zpcjDY{1$}j;e|H&V15Gngo7-5Y%6sq7gCbdL#WnXadwH18@>+z!c01SZIH`%9ihx3$oRS8q5Kfbr^7V?1;m9Ff-R ztN?{_gY#IB@9S~NEH@#W?022^1k0l5TW+I0J9Z8m+WL0=Gk(wragg3!oeY9pWvsA7N@1=I#I{kbTrYMcZs6&>j!#MGm-*p# z<^u4HEqtc+au~<0h1O4`n zRyuL$BO+a$AH$zp@T%Yt;+1fH{(zR>mn&oDHn8_vX|~Y-FHZYP&Vl4F`z|Y@eua+6 z_V17*cP*HlfPL|u8~2C6YCVheWccMMf4-LXe&5r+?Z!Clsf*=?wDkCjgMPl8{64D|j&o@Fiv;sZn62dLrO96*_X++D5UXFCUzr>JLGoRw<~`JA0; z8SU>n!^)C5ni$Wbs9UyT;NhoQH#xvQk`(zl#@F-G`^r|tV-z2>iT3x9U8CF$nW!Je z5?^TuxgdSgq#WhRBHD4^ApezOmze??BfmGPz|Z=KZ}+H>KR@xYc14_4l-O?%z$s$- zsXoZht!VfQ`BhEpFj)b=9#wogfqr~hy;6S*9DAtX_XKuXoDF{mA!oLI$~=a+6@SI) zhCmj&dg7od@~wU9eV`iZi)4y;)xm!2yM|#}eh!sXuK2<(W7#x81l%{6A)5!gBNaIk zwESqLKl|AY9*;{uL3>^lZEuojgX}8p9V~~qUq@L^(Vj2kO7;c3C}&-s%6n>cbQLP_Rban>`0{ zGnUxIh=P~vm9D2jU$EV?C>bnK>^7GNMQ%t5B!FgBa%NhvyWJ_@ZwFqe5$>nWJBwVI z0V6PZ;;Dfi`gMZ)-#a?kA7rxbGKc=$Xf?Yg_-(;5QXTra9c9n2g1j3Z<;EjFcP=_> z`+@Q6Do+H05z`D3PT<9LTeNb(gOc5qyx<$j(Q%sG%k2Bo7tPoClPJ51hkd=~(62^JbSU62=Gwz=Sn->4@ zV?5sW9@r>=bH_W+%f2AxS5x%rQIE50v~$r1Jg?}?nf@2~~md5l^FeS&=PM`+y)>K|op z{2>RjwqvIs_A;m*nv3-U^vWd>OPKah?^{yLt!h3Dj1Y?gSfrLl3q? z?{7}LOa@mPYz{q#zCT(aatrDUd^Ah?1^vYDD3!bLGi|ysF#x)J^*pr;{S4PvT3d^L zJ>yK{+>3ll=5qb>f!<{5SNbQ=FEeh<2b6E#;xgR}PPoxYz6Jk=9WtH<)B7w~Paw|? z^*`=D1{t&*Y=uDM<)klNApKmdqAcuw&V0&^0eh~p+Al!AY}zYs0QvX&JItC7SF1`w-foR+?bZf3O10eb<)8dPut5?K%0!uevQcE{Fq3JIpmfxxv*D| zud}}3IfuAX7@54~Ag8GE2Q#4DDT?gA8KiI5JQ)P8*4?wA@vzV?bu#SFYO$Sq4Q{ax zmhl3wnpUYMAwGljG-V~oozGa0WI;~qaMRxlzgY*@b?bs!?R55e#}Qvu`Y+-n^0iZ& zW!e+Z{WOo3NH5y@+jw>uEVZ)g7x%`w=5c?fAIfdtdspv*-XLS$dUo*PP;sUvC>m_L zITq#TqZjB`K`Cp#k!>e1|8KFXX~F;d=SjRXU^i=yA0x=UPfC%~2Yw1m%T9soD4TtM z!q5I~Ez2xu&o)yvYCRamJ8+W=47k`xGDi8mh*-r#kl%e@?tXxJCdTvoPMyNv?FNq$ zG5d6Fwmit)0QPX(Eit2Byj?ZpC2-V`neqY@`4lav0vcP1Dusd-66cb3A?~+h zva8kLo!)D9a?rmM{N$7b*`|Oo*&OZ8O7d`Mfwq_ZeqIH8B^oc?2e&?BZe~Jz4iTo8 ztx=CuI@L~#lWCJ}S}5X+W)8ii4La9Qf=+^Nb=5cSz<%)VwHrR*71ws&i_nu+L?mn> zUwbg5eF*GfY5t%L{nzvFgigC7ehG@TI@s)7m>vjj*pi*^?}7V3Zq}hz$oyK}nL9A9 zZ<{@=nUH_|8(}|W!4jGIwmPsZg{5l}_JN{FGJW8m>2sS)Kr0z0-*NEi^>~Ik_?fgU zYI6nyy+Ys4!k>z!z1S*bfxFq$nLc@pd{&?8Q~VCu_SWkM7AXI?vAAysxWk>xMW&er&h{K6K*QyfSa(`r`6>-ec=Y6XH<4-;`uz=mT(9ZHL;J zC@)iF3#tSaFYheT0C(+B4i-fJzS9p1ib8JeOLi!O-=S32W)_S??xxH)hmhCNyRsVh z!BrizqtBp!y{4l03-UWqU-IM<^aqXB^q%O8XvF=)90Q9ps&v&&K!D^lqF*wC}@zQym^jqTag; zr&9GGt10vIPNCe$<3*xB;@bYpSbPT5Z&SEmg80nbJY_$DcV#R$dVpz{w>f$u&Vq-G zPiWu&zdL$8yc!H&&)KewIPLTnUz0((rJ(pev{Pt%P^$-g^10`o9Q@vNjj(wFb`||# zm;wt;p0&2Z{$0mD>r;>?^<=EyLgs##Jr{_$gvyNaJ<*@JEVb6t;L@$M7<F%DG54fDRYUV+)OQ2(etf<86q=MR6-Ip9Id z49a&@4=|2^F+63P-oTG7!zM}*$`@?H(zu{+42dbD%}cH^iz92`KVbB8T{~o9qtJ0d z$dXw*jD{eO`KHh8K|5Cu8Omva&uTtS_8@-+zMdMie^241^gWLUejAqKGHL%Vgfonx zVg<68=_x%Qu%+<#O=Yxqh{>-g1>8Sau%!xgW3RQ`4F0s+Wv>TX+x8Ey!XGQss=OxZ zy>?!fc?4P5>5G0pcsk|cuO8^hq<>avH0nQ(2uFK;3U+!*VCB^$@&i!i`lIXV7%yJM zNnsX{^Jz>o0dDLIP(F-uXJ_h*Rmg%4kG|TXT@j(s+ol+AdYuR<+V{^v{Jjhgpl5)J zoG`{Ofk`%kHr`!gDv#R0XNyNDyePN7etb9;tbdcV&lgN7Nfu$F*)zCqBVECLrF+w% z*OzfGzOjYzAmn2Hy{e;-eQLwJmXq*aB1XQz2eMaKf20D)@=l!P4LD+~JmCiW$k&=D zDlcJv_SkUJz84g9eYScZ^?Hu|HV8v`%7qw9O`Nj}IUbZt;=JdY7+`uB+}Rdt-v(Y! ztZChZbKg&IGsZ61{dx81qy+TyN*2Bc!NHUDy@t@=zPZG>0aPb)6Yjy@`oT#SI-JwI zJH}S}aK57__uj35{txpQBM*4KeAgxo=oMn5q@y9r$xyTvA-_1)z-Wzg-JX5K+WU|- zirJdQz{`~>GLrCjF{|m(UU2dJL5WDT&oVSCD~|elc}4@f!0DRYLKDy~$TCe2@r3LW z8XN${Pka7q1`h}Kj5MMAd8X*vanSW8!$AY+w`85Uwy+!XZ%5h7ZVI0N&m_#YQ1QO! zwMIZJXqn-@=Ng#BADg#K#`(&Sg$O1gk8Ev|2H>wLsYn*+Z;VWbcS25Lk&D~`R*!rB zI|*iN%Ko~bjrUX4%c3J-(WXPY-$O5d`^~xt)Z>ZaOAJT5dD2472ha}d=z;+WysjZ+ z*$Dj=|4XTkkh^`8KY43n9kIV8hiD)V&aqd6K*|nw$pc!rcdsvG90xxd>npZ_X$Ag4 zb)cw%g2FQDH5)1N`>Er*JdcQiNbm^HIw3OZr{~sWF++YG-&%?!5T!7-X02i?TaV&*VVc5{-5Px$0qPzv-bCM;_??C*g+!$Q@OG zhzznHzZkX;b_QMfMYka*2GcEcg4uDM#k9OU`Q}^V0t6V)R5)|n}?FGu7GbY6ujs{TpG-LpW7fG*6Q_Y2e)k3{MQVH!KV`=JXpcD^8V0$sY4?f+Wd7$i4|aoxhxS$cLq3lV7mHj1JqLaj z#lkLk|C`VBD35;`!R7$jXwR~Q4d#~#-^6AmaIw3ZZw30xRI$eoVRx`&TK6RK*75H9 zHClb2u#bQLf#*2I#(KfZQihZwaCBqpTq?L*VEIBC+8Z#Ln_`7rrhj&JBlwBo^m;q^ zoA^x3aKhh>h|ecwk$+{W!)znSk91j!1#P~*kLuY-8#m+lTH_zEdsMvli97r~(d1j& z0~QI+zSD<4S)SYT0g&5-e%02)&M+~ekO{J(9eqm^^6q!<+ZRDF+k*Z;5&SmZ&oWzq zJTKk9aU<*#UQfNa2S1Xb-{&7A?=N4H-n|DkDv$G(BhUZ8Ki>SmKPKW|_rJG2+K)H? z-&*`))_*_V{NIl^|M%m~|NVILe?Q**-;Xzw|NHUg|9-srzaMY@@5h_}`|;-ge!Tg= zA8-Ee$D3)t-MmJ8w3<ebfmCW1c4pmZ#=reWIBNbGl-q~$d^8uA` z<-9TXsEkSos-6o>HcDW($LZ`ber2Ky5_ z!-@0vs6^kPQ+|pCR3b^sI4>rLN-V#s%9pxJB^3XSoXE?g5{JBQzjJ{;o$m#=dj^#t zZ7d7jfcUC}nt1{dPss2!^9%5Qx1#IPLCELdxgGa;L?t3*U$riRe4RbX*TKRs=a~h- z>+-2Z-_Rd7v!38Bh(q#8=&^+&DiP_}HyI0xX1oXt&!-alFH8sv#L?_Bx^)))S|_X| z^8oQvFY$Z5hU_D(`iy=Q_vO;)gW&!iI8FRH3fUZWD09dcXAkdLb$aDIP_O2l^h z>R-J{B}Cpf?@4XKl51hOM9#D zB>aUjYpB1xLM1*G@}JhaOeOe*B0s-KrV@rXcLjMuA1^_Ffe93Rb+D}MB9-t88Z+`m zKlk~MyrE)zUaVZne~G*tAq}1GK)Wy7ewJ~7u}K#qn{H5veErE>Z!q}Co*%bU;eX8@ zL3{Yy+$F@ba*0ahoT*LJguGNjX*`igB{a^Te&7hb^6k{%H;D7ri<=c)h`Yb?>Ftlv zRARC4$Nhpx%-;kj#(O9%Z6Pf9QHh(Ukm6F)rx@2Q{n%0EXX-uhhfz7|0x za>bXr?u1hb$`il#sbDIxkRjIg08DIKaCd}Wa;uf$tFu(%iw@=RL-5fjnX++Ea-)B_ zT>$2j>-{@|u#YYsy!bf?{co+cWeGvQ;?uVqLN2joF?<9&u4_|PH+-o?uer}YmorqN zOd-pXDU?bKRbIa<34ixr_U~l%Lq1RKTbxF@Uy>Hymgx{vr3LcmNjk*Q?ddJ`S~^6s z`K*h+nhw!MfpUrg%-1S6a?sQvnzN6lDX8lZI+^lg3($|| zI=y#L)ggZCmhdmZ@5IHl`&*Pzem8{OSwn{y8OraZ5<-3)cjAMiVpFsZJ3T9dhSoD>00R5dpmb68N{)p zzA1AQ6WPL;vsI@%tvC zL*$SgG_S)Ct7@m_ZTM{oavlrx_-Q59)jQG2c)a5Y&yi56~;jM4LU@_ z_2kc`QCc z>JVRN-EW0LKXTn~tmYr~ox6%>SpHCny(tY@=YC^<6rmPZOCygC0dYK%I)u^KgC9@P zkNTDn1%2eVW$5jP4&<{hL!`n?82vl+@$r7-NvNA|;y(H_IUJ&vj(!+Dt^V_#AN^J5 zOD#s84KAlfZb5$9HvfEdghPkO^G*NsZKDoBFOa%b8s*hle+w?5--?wv;p@O_SIgx` zFfN=n)KUU`>fvYn0rk6FDz{VEb%@Z&U-GrwI)rP)hW=vY4hZAXX@+fKi92 zjoWHdhJGG??H{xm@{KQn>v@eZi^#>1!m^G}>>kwZ^%$kQ+v0tWc z;Y@?S1g34@nqmLZrtQNaF!H(eiXig$P?U+jXheKL5f#J9bQ?^r_Ev6R`PsG0gmwFi;f?3V+l`7Vy>eguE%UDM$&)cb{XlHsY!Owwu z#igGoPh+1rQB`+N8$5iky+jE0`qzpYZbQ9G5jV33#IT=4r|-Usb+@ziF3)vD|D zI@OLi%K~p}CW2m$LqqCV&x&b_yT?&aRciOEKKKdIEpShQy;$I7Jwwn;TR+RU0j3pXj9Q_R{@$=3Rg8DU(H|S$b;ywfcVq$auLJv(<%J&3HQ>j!;bm^vuPn^%PxE6Po@JifP=<9V|J(ASH}cil`SH&UkazJ_I&+S3B=do^Zkr9;<=yqW27AII1hafa>f3zZTUO%XbqJJo&N1zf%Eoi z$d51qQ0s|iiDL!!jh|&x2{?c5R*A(w2f_K5dDV8^1DtP7G9V`GUsu%ecqr z7e&F1sT$|q<=A7hko%&Sy)7H4#DmfvNxMf>qVh;}mK)-^-fPnpSBZ0=U$({`$R(qj zM` zsl@Z7K!a$+XL3`_^E&dUFh&S0KwoLkD!U!Bu4lVXe=U`8Ii#Ash<-)gS6$?W-{YR< zV|8H8#*bxt;9u3*khuwQ+6=GUjVqxNQgqZ??~s4(T3@3g_-~CZIm`l{tKaNBT}LGX zes6H3-48tT54F>yzaDYLWm@R}k+TvrJ8%wW(%%%U2WoP?E1bo7e{;xE+CK2IL3GTgHc-bhw+IF@5r-#SJP^_f+b1LOSkSma?WP zX_!Z(bdy+&*Q@bv-Wc?^rKMdl32`;;x%E5(?Zv2D`iX*juUc*XfFol2a z({AFfKyoIcGUq^e#BMnz< zl8}d~`3{B@%(vI#%G|h592{}a+mCZS^S<&;MR6GK{R?|~P=7Pma@!c}pB!m?&6R-p zI(%{WNu2YKHT(a zY|O(Y^O=*i=${>3%GycvM^L4qDjxTcx3|wKha=7hnnwyt5hoj8mx2uL8(syW8`w}j z_fSil4*tG7?6P`|@{V);h7*uiM-~2AAa2f2UfD0hv0lnQ+suVwAFB9f8G(Lk?~i}1 zg8NMpM{JWh?lq1fzU5K4r!ZF37%;+qP|->FMlkvnBX(NQ7x$H|yS_=|{uTbi&e0Wd z4?Sf0o&-M|X(Wqc5oXw=N# z|0|G6>{`ydQiS;aR4@j%pdH;qUt2?PKe@tadwde@HN^6rNCz*;5qn$CP>GcdeE886 zte0oXE)5~DPiT(nYoHJ}Hb^$P)>8;E;?=jIS_D85 zQivH5CFcuu6vA%DgTWb4XnWcEjtUBK;?(A~GT>3mtD^VIDMan?+T63Ke>C-~+utGz zv9h^ZWDNRBdHdG3QVOwM`QClQ5(;sywEIP8F$MdhKi}nh6ryCvHTOp$+PhKcOb3q4 zwV4*)p%8Qm*(R=86e1?>VEMjm3SshvmwhF2efxSXtKf->sW&KPhc$Si)`E9n2&qQTb61*X=8s$igHy+f!~}^?r;XwGw~g< zKZ15e=kEDxz;4a7KprR7QM!xcG!vNCVBj``^;E;at}BW4Zb!Y^VTN^ek^1_!Pb!{6 zLziu1u#Q}ZdOwt79jJ@tow{o|duzyl>gLnl*77=X~K?Tp3{0{rdJogu1je90e~+qbN25QCQSP_TVZWW}s?7`vH-17rx5a7;J?KT0e5!M>uJ`O92@4@_bZO3C8?k?^ zPNa@HV&8aJKb1C!eWTVgr-ljnw0r51!3X;c;&RRd@VWH+W3=}kyc~K(YRF?xfX=SZ zc+WA*`@%yM@%|)7FCPZ4BsOh`M4VfD=7%&e9@TTr4!&TS<7bsrjK?Xtmu5q#H?A^D zqU~GZYRi8+vH!g4;tIT~fqnbcxA9FN!Fg^_9o$0l*?v(Id9ce=tG_#1`fMhMSM6q5Q`l*H4dj;Cjc>${z=lf=M3WAXVsqAu7vsLE@k`=@s;Vu2#P`$9w2@Bfc|N^k0*Mn zVIDpS4$MWr{0plFW5AuN*DEf`>(EBdM6_nbP#kNxJ^yz3D9pC}^o(HZt@O9XE26~cQxV?)Dm_#IAGS7L)K zwBeO(FzRj7eVA|^ek|2?roM-sui-SYPXO)Z%KZxA#rwoR-jcyw7*E%zr4e!LbLyII zI}qpcO~>&S_58N&yk!LY2MU9PE#zBVU%VTBQ;0oYHqp;OW~N_3 z8cP%+P4=?%t6vmi_&C>P7U)AV*$!CDQHVg9+ylZO`)C=z3;OSRsf4$GBlzp<+%yxF z7)f|QYDc@OzeWE}AwEjy`D424v5$u8kEN``d0Ry+uMY7wHde~-S)mZS{}CtrVfXaR z`y&d=_{VYTGu5vVU&0dGbl4&V>rYc{5PGTO8C-V}N1ESU_Y1V&|5a5*d4WP~{X2SQ z3G(wJ0cw7tUf-X5Uq=W4qNLf z#OkERtUS&qck^pq7IEH4x}v=D7k28*XPZKCo+z9f`MMkD68h5?V>aMn$FJ;@s8_dJ zR=XBnY}U2%E`>1dGJhrjyQ=MbKR&!gAq=i2?s{?q`CCp? zsetU9v;HRw*vzY5^*j^pC4|cfft?DS^UdiLLLhG8#a`HptT2i-q*4g4;!39DmnlR= zf$pb^mnekfi@?M0 zENx7r5C=sP3tHkR1kb)b`+P6pJa_ciGrDLBafQ2V=aJ{{`uxqV zp^$yc#V*)_4S6iXf1!V-&^fDyyl&ddC0&Pn`7Qj2Y{NK=c2_*B&!G@YZH=ko(3_HkW)&6-m-&7Pqw`_LSD|_uADH-zM$MXCUbrAN($mj`!mx5 zyPOlLTQ?)m6Sod0I$xv^R!?8Ys~~?5S%Q_+5-0>$QAK7zEasUek-UsNsyANCxBz*1 z3)%8(429r#X*I5a{&)Em-)_hib<_MYD7P$6ip`Cr5Z=c>>Zya~#hT(v7{6ml_Rn*p zDEOzY?Uw67#g|dimr?Gh!D}&qaVg4plv#>#X{)&sayo=UycjywwHESIPcN0FGgyZv zHwUy4@0IWAFY3{*Q#<9@Zj5W~A?lx_@F(>#fLj{=-uf0e-GF?Sao6J~@S~%>{J{c# z^7@D^`Jmgsh9?&O6v8lHVCQpRrZmhc-7$Ye zcaFA+RN{V;ox9zz6!(RYO}8InzC08sY}#PYv2>i3ubx71O80-X#`<1gz|DFF_Cvod zqHkf|9XhdLO)kjcJnV4n0fn&c3zl^#KwLS7yTdS_WW<_x$>&mtM)QDnVXXTG`Wohc zn12RgC95)^u(QZ}KG-jQe?K>kc_;Lc2t5jZy}Z=f56&>1(a-^1#j;CB?of!6e>NH3 z#y-HU7ojeR^;_G`>==srKjc-f#lvn$@}90R+FL8S=Fud0yrhI21AA)avbyzc*qy!A z?+iKZ)BTCC4D2_34P6FU@4;TP-*PekxbMX&?8Z0>@drk1!F=QHIez&r=BwRF+qp-W zhx0vCDN`8NhU)I9_c567p@PHB7}w0^Q4Ys^?B8z)!K>)^ShpijO%{bn-eu4df;hi9 zsF>6Ed6m4S4DE5PTX=aF?J2lTW|my15XP(%nj=^zgV%BxdqLaLrVP?mtdCMF)hetj zw^#em9Kbr+wZ1dwDE1lkxny!K)(QP}6=vEx@nU$P@B{VK+&`} zz6d&>&WgzghuW`HJw-nTQ!An?u@7x$3J6oidfHJ_H)|1r^>DQEN(t7(x!+@7#=&u} zUe0pl!Jhrj$yKbMrQ`=6E}`G*<_<3!VNZ_nmz=>o&g6gkh5_qjb$_eJUhG@yR*Mza zFdsjfSgg?IdxMCJw{aNq?NTdhfc0|9bU-KjEcSi>wI@=+0n>};jeOCM;Y~hX=-0Z< zj zy;wKZrctJLX#cD8!6`E0BcGWY=|lPXst}>Y01A;c*5?xj{mHFz zYsZ7J53fJP7VSqN?i76cwirw%I9Z-{4~3BlzwV8Jl4r@p;2WVwxdCLN^UcrtJ$__j z&qe7|oS$n9wigfVr%NO z&B%oB$gZ}J24o^TbKt%jnM{nkN=pnQu3odpbQWm0uyBMb8%QP&f8ysn3cW@56R#P> z8Pwczt0;s_o9SW5e*237L3Y`=GZC{TMti(dA4f6Mb@~!6YtJ;>MK%qB@~GBI3&e zPEc-8ZAqC-1P>o-tyCcsJZ@GCD-<#z_5G~Or(I+s-Jts*MGgIbW<4>5dVM7$c28jE z-2CKLsU(?@7cXq@6eAPOVhJvep!OvV{haM&!YypkRd+L)I8g1JO2_9{I0o?l7a2Hpsc zT04q;{Qm0Hg*2SgwCCjY((zn~8}Ovh!oL4tphwQ45&P#W;=?lb|C8Fsep^6)iJB1M zjD6nYZN1ipGVCK9t;gm|v9F9d7iObeZ7QX#4DFrzt>`jRf&J&cbooBSaYEr5eJR?z zLVsTO73!I_Zdcw=L?Qn8eRK>f#yL#?+M80yrc7R%bdWc7SRS;;z8*i%a3vl4`U_HR z#1ZVPi=8V>dWbjrTxYi<`1PyKPRv^GSHD%4iShYa&tp2!3Bwmkt<% zZXZg%vO{mE)A}J0?XcEO9A1U2FuU)5FXU&@^|RK9*X!Ac#2lUj?zt;XzYzDPGg`9t zu#+#&+jtgnP|wOmzCrowu^$gV;(4~|#7%~+crNYR^NeJGymGxwv37w!KhHI+o$#l$ zXU{)(^t=9rXq+(4VUC$sr5Vr%+EZz^^4ptKp-(mOW@HsqJt=L^cv7hVfPw4(bTw@&k;TI6c)h{o9 zxuJcR#+vGfXm^I+Em|9Xy?NIZyu$OqVxu%y0Q|1Z$S&@HeEQOvw6-MN`wl8vMqGsc zFp7BeWg?jiY?+jJ+JMZU)P-W!V zM*j;Ff#*1%cHo|$h>NP4RHuggth-*|!G`D4g?+D64Ir=YZ7qw#^Y$f~v2X@)i8X5- ze*wQb7a5-LA}>x`KEBpOUIKeE|n#+lRz3-W>x~Gtb+`QFYP>;2$XlVuU7vGxN$L){#)^m*Fd zwM7W}p@&5~V$YHY{c%kRZ}^e-QoNn%Pa+=Vzs^p8T#(lATlF}JuYcv#(UzjYQn#K9IcMlSB+&?r_sSMk02mNFILW2!9f`Yf_Jph{SJ?q-I=5gur)| zUq>9^CvDYg1p4ykrW&n7B%;vfZ^hvQBtj>_?|h>kiI81?*k4?SMAV${txhGAh(J;G zx3|==-!d*I+n{|H!SJJ*r%42_?3o~G#3dH^*}V>aqmLN5MxmWw6S?0y;oo<$_(VDM z=i?31TF}nkN|)#&3lb5x>bBnj`LUOtx}30|L_ADAs#|PIA|yBUzi0uy1f|ods8{^c zgIfdT<+~18aaxfGmO<09HK=bp%S1k4Kq7w5-wGU|k_gKA#O67~`^8E4ks|c7C2h&G zkUix^iUc%Bgx|YFFs6x#6D$<{=T| z|BJmZ4X5f0<3$>b6$u$qXi#M4ID;cY=8BYAG9)D_p;S~TO-Lk}LW2gRln{#s%2WxN zN|`x?GEeu{eV+T=5BJ^=_xu0TdiLIHuk{YU_xHZ<-pBUr`}z<`2Kizz82D_M#?nzU``fV@x z3+W8v->+B4f_@SU_n)EQd^^+AKwN=AhKD(4BBbCKp8`)!>}&Xb&~){3*o*3X(1M#m zC_%j+w?U5K9KR43i~%2>DBfHRxfarm8=*(~HuKBD81F7u%bCogcTvzV8QlR;t+zgpfnM<BRhKj^xY&29Zh`Y?0t#kUM$q zdDFPQa6C6oVUkWlZeG_r0(nlByG~3%{}+>kQJb-j`>BEVXD8^yEStV^=`@|R?HE`w zhWQHCndQkiuf0`5%4L|2=iw0!fBWbpP}+G}STCJKlyTF(vgl+(UMgb=juUlF`PM2Q1eAl@cz}~qbQpaN<59Lhwp@_x6i!N97#4Me#!Zh^nWBm^YDz9+3AI^MrJ9}oCVC+w4}hMRimB)#riiu@p*SX$lu=7D__Klx3+>Y|g$dm9eABVTlP znTpS&Z`m|;qGvoF{ZG#hFKOfh?xfJ0)Fkj;_|OwJ@U+Qd-9Ixp-TtrtthJr;n+ zg$p!JoxTMAJK>(W4fFQ&Ki^vqezYzhVNk(qDWfH;{lI^VdK?z)3&4A?K4Aex@LK=1 z9nl9d`UI|9_OCz4y(h!i1-^7q>D|YM@!FHyd6l@YKz-U(19kn{!#6hQQD&V%#SQZSuOBBWG4vn~49t{jPPnHi-+|;gq>an-pG4+VltW6cdFtqt~In z|Jro$wIu3q4aIF2gtW=htnd|0m>=-hYq?Aqj*VD#XXX$HwO-^tc7#ULBMf38Xv)FyTB_~dT# zYZHq<%bio^8RWCR0ect5Lc;SkZ5(-JrHARx8SvnN9Pi0d2Jt)CoSV{*_{;CDKCx1p zJUnBvIt2R`GH;*oUx2ah#w$+jXR&X5d}x+I*728C{u*Ww>sw}ud&a=4QWqoCaeur9 zo6FPR3^Hq98z$QezJIO$Caa4<2J8dnH;*t#^+q