From 91d59ee2a583848559b99fb8fc429366813a38f6 Mon Sep 17 00:00:00 2001 From: Melissa DeLucchi <113376043+delucchi-cmu@users.noreply.github.com> Date: Wed, 5 Mar 2025 15:18:17 -0500 Subject: [PATCH] Re-generate test data, and update expectations. (#467) --- tests/conftest.py | 14 ++++++++------ tests/data/generate_data.ipynb | 10 +++++++++- .../dataset/Norder=0/Dir=0/Npix=11.parquet | Bin 8913 -> 5089 bytes tests/data/small_sky/dataset/_common_metadata | Bin 3983 -> 983 bytes tests/data/small_sky/dataset/_metadata | Bin 5103 -> 2094 bytes tests/data/small_sky/point_map.fits | Bin 50339520 -> 100670400 bytes tests/data/small_sky/properties | 6 +++--- .../dataset/Norder=1/Dir=0/Npix=44.parquet | Bin 7240 -> 3425 bytes .../dataset/Norder=1/Dir=0/Npix=45.parquet | Bin 6993 -> 3178 bytes .../dataset/Norder=1/Dir=0/Npix=46.parquet | Bin 7234 -> 3419 bytes .../dataset/Norder=1/Dir=0/Npix=47.parquet | Bin 6761 -> 2946 bytes .../small_sky_order1/dataset/_common_metadata | Bin 3983 -> 983 bytes tests/data/small_sky_order1/dataset/_metadata | Bin 8420 -> 5384 bytes tests/data/small_sky_order1/point_map.fits | Bin 8640 -> 8640 bytes tests/data/small_sky_order1/properties | 6 +++--- .../dataset/_common_metadata | Bin 2224 -> 2224 bytes .../dataset/_metadata | Bin 2645 -> 2641 bytes .../dataset/index/part.0.parquet | Bin 3867 -> 3547 bytes .../data/small_sky_order1_id_index/properties | 6 +++--- .../dataset/Norder=0/Dir=0/Npix=4.parquet | Bin 8247 -> 3331 bytes .../dataset/Norder=0/Dir=0/Npix=7.parquet | Bin 0 -> 3393 bytes .../dataset/Norder=0/Dir=0/Npix=8.parquet | Bin 0 -> 3331 bytes .../dataset/Norder=1/Dir=0/Npix=44.parquet | Bin 8428 -> 3568 bytes .../dataset/Norder=1/Dir=0/Npix=45.parquet | Bin 8470 -> 3593 bytes .../dataset/Norder=1/Dir=0/Npix=46.parquet | Bin 8371 -> 3585 bytes .../dataset/Norder=1/Dir=0/Npix=47.parquet | Bin 8336 -> 3561 bytes .../dataset/_common_metadata | Bin 5132 -> 1274 bytes .../small_sky_order1_margin/dataset/_metadata | Bin 12347 -> 11273 bytes .../partition_info.csv | 2 ++ tests/data/small_sky_order1_margin/properties | 10 +++++----- .../dataset/Norder=0/Dir=0/Npix=4.parquet | Bin 11523 -> 6173 bytes .../dataset/Norder=1/Dir=0/Npix=47.parquet | Bin 137676 -> 132250 bytes .../dataset/Norder=2/Dir=0/Npix=176.parquet | Bin 29117 -> 23711 bytes .../dataset/Norder=2/Dir=0/Npix=177.parquet | Bin 89086 -> 83662 bytes .../dataset/Norder=2/Dir=0/Npix=178.parquet | Bin 95714 -> 90290 bytes .../dataset/Norder=2/Dir=0/Npix=179.parquet | Bin 103119 -> 97695 bytes .../dataset/Norder=2/Dir=0/Npix=180.parquet | Bin 43359 -> 37949 bytes .../dataset/Norder=2/Dir=0/Npix=181.parquet | Bin 56287 -> 50867 bytes .../dataset/Norder=2/Dir=0/Npix=182.parquet | Bin 75097 -> 69678 bytes .../dataset/Norder=2/Dir=0/Npix=183.parquet | Bin 69687 -> 64268 bytes .../dataset/Norder=2/Dir=0/Npix=184.parquet | Bin 82739 -> 77320 bytes .../dataset/Norder=2/Dir=0/Npix=185.parquet | Bin 167156 -> 161730 bytes .../dataset/Norder=2/Dir=0/Npix=186.parquet | Bin 32523 -> 27117 bytes .../dataset/Norder=2/Dir=0/Npix=187.parquet | Bin 44887 -> 39477 bytes .../small_sky_source/dataset/_common_metadata | Bin 5490 -> 1335 bytes tests/data/small_sky_source/dataset/_metadata | Bin 28990 -> 24504 bytes tests/data/small_sky_source/point_map.fits | Bin 50339520 -> 100670400 bytes tests/data/small_sky_source/properties | 6 +++--- .../dataset/_common_metadata | Bin 2296 -> 2296 bytes .../dataset/_metadata | Bin 2725 -> 2721 bytes .../dataset/index/part.0.parquet | Bin 4153 -> 3825 bytes .../small_sky_source_object_index/properties | 6 +++--- .../dataset/_common_metadata | Bin 556 -> 556 bytes .../dataset/_metadata | Bin 2913 -> 2897 bytes .../dataset/Norder=1/Dir=0/Npix=0.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=12.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=16.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=20.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=24.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=28.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=32.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=36.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=4.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=40.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=44.parquet | Bin 2881 -> 1502 bytes .../dataset/Norder=1/Dir=0/Npix=8.parquet | Bin 2881 -> 1502 bytes .../data/square_map/dataset/_common_metadata | Bin 1612 -> 661 bytes tests/data/square_map/dataset/_metadata | Bin 8773 -> 7762 bytes tests/data/square_map/properties | 6 +++--- .../margin_cache/test_margin_catalog.py | 1 + 70 files changed, 43 insertions(+), 30 deletions(-) create mode 100644 tests/data/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=7.parquet create mode 100644 tests/data/small_sky_order1_margin/dataset/Norder=0/Dir=0/Npix=8.parquet diff --git a/tests/conftest.py b/tests/conftest.py index 78891cda..060d9a40 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -116,6 +116,7 @@ def margin_catalog_info(margin_cache_catalog_info_data) -> TableProperties: def small_sky_schema() -> pa.Schema: return pa.schema( [ + pa.field("_healpix_29", pa.int64()), pa.field("id", pa.int64()), pa.field("ra", pa.float64()), pa.field("dec", pa.float64()), @@ -124,7 +125,6 @@ def small_sky_schema() -> pa.Schema: pa.field("Norder", pa.uint8()), pa.field("Dir", pa.uint64()), pa.field("Npix", pa.uint64()), - pa.field("_healpix_29", pa.int64()), ] ) @@ -133,6 +133,7 @@ def small_sky_schema() -> pa.Schema: def small_sky_source_schema() -> pa.Schema: return pa.schema( [ + pa.field("_healpix_29", pa.int64()), pa.field("source_id", pa.int64()), pa.field("source_ra", pa.float64()), pa.field("source_dec", pa.float64()), @@ -145,7 +146,6 @@ def small_sky_source_schema() -> pa.Schema: pa.field("Norder", pa.uint8()), pa.field("Dir", pa.uint64()), pa.field("Npix", pa.uint64()), - pa.field("_healpix_29", pa.int64()), ] ) @@ -166,18 +166,18 @@ def association_catalog_schema() -> pa.Schema: def margin_catalog_schema() -> pa.Schema: return pa.schema( [ + pa.field("_healpix_29", pa.int64()), pa.field("id", pa.int64()), pa.field("ra", pa.float64()), pa.field("dec", pa.float64()), pa.field("ra_error", pa.int64()), pa.field("dec_error", pa.int64()), - pa.field("Norder", pa.uint8()), - pa.field("Dir", pa.uint64()), - pa.field("Npix", pa.uint64()), - pa.field("_healpix_29", pa.int64()), pa.field("margin_Norder", pa.uint8()), pa.field("margin_Dir", pa.uint64()), pa.field("margin_Npix", pa.uint64()), + pa.field("Norder", pa.uint8()), + pa.field("Dir", pa.uint64()), + pa.field("Npix", pa.uint64()), ] ) @@ -196,6 +196,8 @@ def catalog_path(test_data_dir) -> str: def margin_catalog_pixels() -> list[HealpixPixel]: return [ HealpixPixel(0, 4), + HealpixPixel(0, 7), + HealpixPixel(0, 8), HealpixPixel(1, 44), HealpixPixel(1, 45), HealpixPixel(1, 46), diff --git a/tests/data/generate_data.ipynb b/tests/data/generate_data.ipynb index 3e1c9032..9e89648e 100644 --- a/tests/data/generate_data.ipynb +++ b/tests/data/generate_data.ipynb @@ -32,6 +32,7 @@ "import tempfile\n", "from pathlib import Path\n", "from dask.distributed import Client\n", + "from hats.io.file_io import remove_directory\n", "\n", "tmp_path = tempfile.TemporaryDirectory()\n", "tmp_dir = tmp_path.name\n", @@ -61,6 +62,7 @@ "metadata": {}, "outputs": [], "source": [ + "remove_directory(\"./small_sky\")\n", "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", " args = ImportArguments(\n", " input_path=Path(hats_import_dir) / \"small_sky\",\n", @@ -97,6 +99,7 @@ "metadata": {}, "outputs": [], "source": [ + "remove_directory(\"./small_sky_order1\")\n", "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", " args = ImportArguments(\n", " input_path=Path(hats_import_dir) / \"small_sky\",\n", @@ -126,6 +129,7 @@ "metadata": {}, "outputs": [], "source": [ + "remove_directory(\"./small_sky_order1_id_index\")\n", "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", " args = IndexArguments(\n", " input_catalog_path=\"small_sky\",\n", @@ -165,6 +169,7 @@ "metadata": {}, "outputs": [], "source": [ + "remove_directory(\"./small_sky_order1_margin\")\n", "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", " args = MarginCacheArguments(\n", " margin_threshold=7200,\n", @@ -241,6 +246,7 @@ "metadata": {}, "outputs": [], "source": [ + "remove_directory(\"./small_sky_source\")\n", "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", " args = ImportArguments(\n", " input_path=Path(hats_import_dir) / \"small_sky_source\",\n", @@ -284,6 +290,7 @@ "metadata": {}, "outputs": [], "source": [ + "remove_directory(\"./small_sky_source_object_index\")\n", "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", " args = IndexArguments(\n", " input_catalog_path=\"small_sky_source\",\n", @@ -321,6 +328,7 @@ "metadata": {}, "outputs": [], "source": [ + "remove_directory(\"./square_map\")\n", "with tempfile.TemporaryDirectory() as pipeline_tmp:\n", " csv_file = Path(pipeline_tmp) / \"square_map.csv\"\n", " value_frame.to_csv(csv_file, index=False)\n", @@ -360,7 +368,7 @@ ], "metadata": { "kernelspec": { - "display_name": "demo", + "display_name": "hats", "language": "python", "name": "python3" }, diff --git a/tests/data/small_sky/dataset/Norder=0/Dir=0/Npix=11.parquet b/tests/data/small_sky/dataset/Norder=0/Dir=0/Npix=11.parquet index af44a7610433db34d1fbcb4f0bbeb0ec6f2d01cb..3d9a9803f6295b842a1e8337ab123f8b345b9b00 100644 GIT binary patch delta 1043 zcmZuxOH30{6rCxZc3QzsXj-Q@u~_3MBAph2P!cqErXO15*YGPb!P<}lmY*1)x@g=O zmzZZ
1UuND!KAkZ4Lo!%g
zx9^&7o9i8KYb34_pN<-l+pdu}&n)o57u0u)kNFFHF;n%7FJ2c4Jh^IXAl+gI@rfDo
zRW$HyAopx4aoGEOF0E+fMRYI>&RwD|m$8$J8A!=@ib=
zgkP3A;b(*TStW{q
literal 8913
zcmc&a30PBC){hV{5zrte21QMyMAVm;uu8-wFM&WJi+~V<3ML_tND{U{h)PRsUF!U<
zzwJ=Bs&&*l)}^D?RvmGvtyNL0wQ6-_rY@*|{jJqjZL9yeFCh>D+WAKNKRxfBd(OG{
zEce`V?~9Hnvm)7a_BIxK7bBHDo5sd^VORr$afpWb`Ovw(e%
vDZTsO#LhwQ@g^1QB;K5upTBI*$HXVEeKf;R
z`3bS~{e~wY!wwQr%ZqdJS05*A|eSqy_G?P+`5=ZS}1>T`--{C>Sn<{o7P_
z76aI?V08I%_qbg)CES-dMzL|FDXn~@FHlPoVUPAuz8nwiut5#CHP
(qL6wYo-a
zP<5(aU8{VmL0zY=S2w7Q>PEFm-J~|Fo7F99i`uGgRkx|z)g5Y^x>Mby8r9wE9(AwU
zuI^L!s~xII?Nkq_U23;_P(7sfsE5@f>QVKW+N<`d$5pdxQBSBR)l+J}dRjfBo>d3b
zbLx5Zf_hQCq+V7B)gkqYdR4uqURQ6ZH`QC}uzFj)quy0V)O+fE^?^F7K2#s6kJT~t
ziTYH1rao6+s4vx5>bUw^eWSirC)9W9d-a1lseV*Hsh`y;^^5vd{iaT<-_;-LKkM|j
zzd=C&w1q$jf_BgzIzUJ01f3xmLLd~vARM|tSLg=ap$C|&^<)+aQP2x|Lm%i1{U91F
zh=Kkv00zP!7z{&TD8#}r7!GkT0!D%Z;$ajdz-Sl)V__Ve0f{gkCcs3P1e0M3Oob#!
zhG{SzQXmy(z)VPkbeILRAp_>XT$l%$a3*9yHsrv3SO8~1E-ZwzVG%5bbKqQ90!zUO
zd5{kU;DTjP2u0wA^I$m?LkW~Z8I(f>c%TxzPz5XCe5i&RxBxDMm2eSU43|JHTnd-L
zDp(Dd!xgXw*20x=6|95xa5Y>58=wyA;ac!P16&8!!ws+zZiG#66KsZ?;TG5eTj5r?
z4Q_`!U>n>CcR?fE4fnvkupRD$`(Xz(!A^JpcEN6V5FUa(@Gv|AkHTZH7xuy9&