Improve reading/writing of fits file.

src/hats/catalog/dataset/table_properties.py

@@ -130,12 +130,13 @@ def space_delimited_list(cls, str_value: str) -> List[str]:
         if isinstance(str_value, str):
             # Split on a few kinds of delimiters (just to be safe), and remove duplicates
             return list(filter(None, re.split(";| |,|\n", str_value)))
-        return str_value
+        ## Convert empty strings and empty lists to None
+        return str_value if str_value else None
 
     @field_serializer("default_columns", "extra_columns")
     def serialize_as_space_delimited_list(self, str_list: Iterable[str]) -> str:
         """Convert a python list of strings into a space-delimited string."""
-        if str_list is None:
+        if str_list is None or len(str_list) == 0:
             return None
         return " ".join(str_list)
 
@@ -169,6 +170,7 @@ def check_allowed_and_required(self) -> Self:
         return self
 
     def copy_and_update(self, **kwargs):
+        """Create a validated copy of these table properties, updating the fields provided in kwargs."""
         new_properties = self.model_copy(update=kwargs)
         TableProperties.model_validate(new_properties)
         return new_properties

src/hats/catalog/healpix_dataset/healpix_dataset.py

@@ -178,11 +178,10 @@ def filter_by_moc(self, moc: MOC) -> Self:
 
         Returns:
             A new catalog with only the pixels that overlap with the moc. Note that we reset the total_rows
-        to None, as updating would require a scan over the new pixel sizes."""
+        to 0, as updating would require a scan over the new pixel sizes."""
         filtered_tree = filter_by_moc(self.pixel_tree, moc)
         filtered_moc = self.moc.intersection(moc) if self.moc is not None else None
-        filtered_catalog_info = self.catalog_info.model_copy()
-        filtered_catalog_info.total_rows = None
+        filtered_catalog_info = self.catalog_info.copy_and_update(total_rows=0)
         return self.__class__(filtered_catalog_info, filtered_tree, moc=filtered_moc, schema=self.schema)
 
     def align(

src/hats/io/file_io/file_io.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import tempfile
-import warnings
 from collections.abc import Generator
 from pathlib import Path
 from typing import Any, Tuple
@@ -228,17 +227,7 @@
         with map_file_pointer.open("rb") as _map_file:
             map_data = _map_file.read()
             _tmp_file.write(map_data)
-            map_fits_image = hp.read_map(_tmp_file.name, nest=True, h=True)
-            header_dict = dict(map_fits_image[1])
-            if header_dict["ORDERING"] != "NESTED":
-                warnings.warn(
-                    "point_map.fits file written in RING ordering, due to "
-                    "https://github.com/astronomy-commons/hats/issues/271. "
-                    "Converting to NESTED."
-                )
-                map_fits_image = hp.read_map(_tmp_file.name)
-                return map_fits_image
-            return map_fits_image[0]
+            return hp.read_map(_tmp_file.name, nest=True)
 
 
 def write_fits_image(histogram: np.ndarray, map_file_pointer: str | Path | UPath):
@@ -252,7 +241,7 @@
     map_file_pointer = get_upath(map_file_pointer)
     with tempfile.NamedTemporaryFile() as _tmp_file:
         with map_file_pointer.open("wb") as _map_file:
-            hp.write_map(_tmp_file.name, histogram, overwrite=True, dtype=np.int64, nest=True)
+            hp.write_map(_tmp_file.name, histogram, overwrite=True, dtype=np.int32, nest=True, coord="CEL")
             _map_file.write(_tmp_file.read())
 
 

tests/data/small_sky/partition_info.csv

@@ -1,2 +1,2 @@
-Norder,Npix,Dir
-0,11,0
+Norder,Npix
+0,11

tests/data/small_sky/properties

@@ -1,7 +1,14 @@
-obs_collection = small_sky
-dataproduct_type = object
-hats_col_j2000_ra = ra
-hats_col_j2000_dec = dec
-hats_max_rows = 1000
-hats_nrows = 131
-hats_order = 0
+#HATS catalog
+obs_collection=small_sky
+dataproduct_type=object
+hats_nrows=131
+hats_col_j2000_ra=ra
+hats_col_j2000_dec=dec
+hats_max_rows=1000000
+hats_order=0
+moc_sky_fraction=0.08333
+hats_builder=hats-import v0.3.6.dev22+g6346e4d
+hats_creation_date=2024-09-30T13\:32UTC
+hats_estsize=98328
+hats_release_date=2024-09-18
+hats_version=v0.1

tests/data/small_sky_order1/partition_info.csv

@@ -1,5 +1,5 @@
-Norder,Npix,Dir
-1,44,0
-1,45,0
-1,46,0
-1,47,0
+Norder,Npix
+1,44
+1,45
+1,46
+1,47

tests/data/small_sky_order1/properties

@@ -1,7 +1,14 @@
-obs_collection = small_sky_order1
-dataproduct_type = object
-hats_col_j2000_ra = ra
-hats_col_j2000_dec = dec
-hats_max_rows = 1000
-hats_nrows = 131
-hats_order = 0
+#HATS catalog
+obs_collection=small_sky_order1
+dataproduct_type=object
+hats_nrows=131
+hats_col_j2000_ra=ra
+hats_col_j2000_dec=dec
+hats_max_rows=1000000
+hats_order=1
+moc_sky_fraction=0.08333
+hats_builder=hats-import v0.3.6.dev22+g6346e4d
+hats_creation_date=2024-09-30T13\:32UTC
+hats_estsize=48
+hats_release_date=2024-09-18
+hats_version=v0.1

tests/data/small_sky_order1_id_index/properties

@@ -1,7 +1,11 @@
-obs_collection = small_sky_order1_id_index
-dataproduct_type = index
-hats_max_rows = 1000
-hats_nrows = 131
-hats_order = 0
+#HATS catalog
+obs_collection=small_sky_order1_id_index
+dataproduct_type=index
+hats_nrows=131
+hats_primary_table_url=small_sky
 hats_index_column=id
-hats_primary_table_url=small_sky
+hats_builder=hats-import v0.3.6.dev22+g6346e4d
+hats_creation_date=2024-10-01T13\:05UTC
+hats_estsize=8
+hats_release_date=2024-09-18
+hats_version=v0.1

tests/data/small_sky_order1_margin/partition_info.csv

@@ -1,6 +1,6 @@
-Norder,Npix,Dir
-0,4,0
-1,44,0
-1,45,0
-1,46,0
-1,47,0
+Norder,Npix
+0,4
+1,44
+1,45
+1,46
+1,47

tests/data/small_sky_order1_margin/properties

@@ -1,9 +1,15 @@
-obs_collection = small_sky_order1_margin
-dataproduct_type = margin
-hats_col_j2000_ra = ra
-hats_col_j2000_dec = dec
-hats_max_rows = 1000
-hats_nrows = 28
-hats_order = 0
+#HATS catalog
+obs_collection=small_sky_order1_margin
+dataproduct_type=margin
+hats_nrows=28
+hats_col_j2000_ra=ra
+hats_col_j2000_dec=dec
 hats_primary_table_url=small_sky_order1
-hats_margin_threshold=7200
+hats_margin_threshold=7200.0
+hats_order=1
+moc_sky_fraction=0.16667
+hats_builder=hats-import v0.3.6.dev22+g6346e4d
+hats_creation_date=2024-09-30T13\:32UTC
+hats_estsize=57
+hats_release_date=2024-09-18
+hats_version=v0.1

tests/data/small_sky_source/partition_info.csv

@@ -1,15 +1,15 @@
-Norder,Npix,Dir
-0,4,0
-1,47,0
-2,176,0
-2,177,0
-2,178,0
-2,179,0
-2,180,0
-2,181,0
-2,182,0
-2,183,0
-2,184,0
-2,185,0
-2,186,0
-2,187,0
+Norder,Npix
+0,4
+1,47
+2,176
+2,177
+2,178
+2,179
+2,180
+2,181
+2,182
+2,183
+2,184
+2,185
+2,186
+2,187

tests/data/small_sky_source/properties

@@ -1,7 +1,14 @@
-obs_collection = small_sky_source
-dataproduct_type = source
-hats_col_j2000_ra = source_ra
-hats_col_j2000_dec = source_dec
-hats_max_rows = 1000
-hats_nrows = 17161
-hats_order = 0
+#HATS catalog
+obs_collection=small_sky_source
+dataproduct_type=source
+hats_nrows=17161
+hats_col_j2000_ra=source_ra
+hats_col_j2000_dec=source_dec
+hats_max_rows=3000
+hats_order=2
+moc_sky_fraction=0.16667
+hats_builder=hats-import v0.3.6.dev22+g6346e4d
+hats_creation_date=2024-09-30T13\:33UTC
+hats_estsize=99328
+hats_release_date=2024-09-18
+hats_version=v0.1

tests/data/small_sky_source_object_index/properties

@@ -1,7 +1,11 @@
-obs_collection = small_sky_source_object_index
-dataproduct_type = index
-hats_max_rows = 1000
-hats_nrows = 148
-hats_order = 0
+#HATS catalog
+obs_collection=small_sky_source_object_index
+dataproduct_type=index
+hats_nrows=148
+hats_primary_table_url=small_sky_source
 hats_index_column=object_id
-hats_primary_table_url=small_sky_source
+hats_builder=hats-import v0.3.6.dev22+g6346e4d
+hats_creation_date=2024-09-30T13\:44UTC
+hats_estsize=10
+hats_release_date=2024-09-18
+hats_version=v0.1

tests/data/small_sky_to_small_sky_order1/partition_info.csv

@@ -1,2 +1,2 @@
-Norder,Npix,Dir
-0,11,0
+Norder,Npix
+0,11

tests/hats/catalog/test_catalog.py

@@ -145,7 +145,7 @@ def test_cone_filter(small_sky_order1_catalog):
     assert filtered_pixels == [HealpixPixel(1, 44)]
 
     assert (1, 44) in filtered_catalog.pixel_tree
-    assert filtered_catalog.catalog_info.total_rows is None
+    assert filtered_catalog.catalog_info.total_rows == 0
 
     assert filtered_catalog.moc is not None
     cone_moc = MOC.from_cone(
@@ -200,7 +200,7 @@ def test_polygonal_filter(small_sky_order1_catalog):
     assert len(filtered_pixels) == 1
     assert filtered_pixels == [HealpixPixel(1, 46)]
     assert (1, 46) in filtered_catalog.pixel_tree
-    assert filtered_catalog.catalog_info.total_rows is None
+    assert filtered_catalog.catalog_info.total_rows == 0
     assert filtered_catalog.moc is not None
     ra, dec = np.array(polygon_vertices).T
     polygon_moc = MOC.from_polygon(
@@ -291,7 +291,7 @@ def test_box_filter_ra(small_sky_order1_catalog):
     assert (1, 44) in filtered_catalog.pixel_tree
     assert (1, 46) in filtered_catalog.pixel_tree
     assert len(filtered_catalog.pixel_tree.pixels[1]) == 2
-    assert filtered_catalog.catalog_info.total_rows is None
+    assert filtered_catalog.catalog_info.total_rows == 0
 
     assert filtered_catalog.moc is not None
     box_moc = generate_box_moc(ra=ra, dec=None, order=small_sky_order1_catalog.get_max_coverage_order())
@@ -310,7 +310,7 @@ def test_box_filter_wrapped_ra(small_sky_order1_catalog):
     assert (1, 44) in filtered_catalog.pixel_tree
     assert (1, 45) in filtered_catalog.pixel_tree
     assert len(filtered_catalog.pixel_tree.pixels[1]) == 2
-    assert filtered_catalog.catalog_info.total_rows is None
+    assert filtered_catalog.catalog_info.total_rows == 0
 
 
 def test_box_filter_ra_divisions_edge_cases(small_sky_order1_catalog):
@@ -377,14 +377,14 @@ def test_box_filter_dec(small_sky_order1_catalog):
     filtered_catalog = small_sky_order1_catalog.filter_by_box(dec=(10, 20))
     assert len(filtered_catalog.get_healpix_pixels()) == 0
     assert len(filtered_catalog.pixel_tree) == 0
-    assert filtered_catalog.catalog_info.total_rows is None
+    assert filtered_catalog.catalog_info.total_rows == 0
 
     filtered_catalog_1 = small_sky_order1_catalog.filter_by_box(dec=(-10, 10))
     filtered_pixels_1 = filtered_catalog_1.get_healpix_pixels()
     assert filtered_pixels_1 == [HealpixPixel(1, 47)]
     assert (1, 47) in filtered_catalog_1.pixel_tree
     assert len(filtered_catalog_1.pixel_tree) == 1
-    assert filtered_catalog_1.catalog_info.total_rows is None
+    assert filtered_catalog_1.catalog_info.total_rows == 0
 
     filtered_catalog_2 = small_sky_order1_catalog.filter_by_box(dec=(-30, -20))
     filtered_pixels_2 = filtered_catalog_2.get_healpix_pixels()
@@ -393,7 +393,7 @@ def test_box_filter_dec(small_sky_order1_catalog):
     assert (1, 46) in filtered_catalog_2.pixel_tree
     assert (1, 47) in filtered_catalog_2.pixel_tree
     assert len(filtered_catalog_2.pixel_tree) == 3
-    assert filtered_catalog_2.catalog_info.total_rows is None
+    assert filtered_catalog_2.catalog_info.total_rows == 0
 
 
 def test_box_filter_ra_and_dec(small_sky_order1_catalog):
@@ -407,7 +407,7 @@ def test_box_filter_ra_and_dec(small_sky_order1_catalog):
     assert (1, 46) in filtered_catalog.pixel_tree
     assert (1, 47) in filtered_catalog.pixel_tree
     assert len(filtered_catalog.pixel_tree.pixels[1]) == 2
-    assert filtered_catalog.catalog_info.total_rows is None
+    assert filtered_catalog.catalog_info.total_rows == 0
 
     # Check that the previous filter is the same as intersecting the ra and dec filters
     filtered_catalog_ra = small_sky_order1_catalog.filter_by_box(ra=(280, 300))