From ea32f7d8036dd648d57e8e2828dba397ef8829aa Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Tue, 11 Apr 2023 10:05:13 -0500 Subject: [PATCH 1/7] reduce 0-d or length 1 header arrays to scalar --- yt/frontends/gadget/data_structures.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/yt/frontends/gadget/data_structures.py b/yt/frontends/gadget/data_structures.py index 8f6880a1b04..2ed1449f25a 100644 --- a/yt/frontends/gadget/data_structures.py +++ b/yt/frontends/gadget/data_structures.py @@ -629,6 +629,19 @@ def _get_hvals(self): if "Parameters" in handle: hvals.update((str(k), v) for k, v in handle["/Parameters"].attrs.items()) handle.close() + + # ensure that 1-element arrays are reduced to scalars + updated_hvals = {} + for hvalname, value in hvals.items(): + if isinstance(value, np.ndarray): + if value.ndim == 0: + mylog.warning(f"Casting 0d {hvalname} to float.") + updated_hvals[hvalname] = float(value) + elif value.ndim == 1 and value.size == 1: + mylog.warning(f"Reducing length-1 array {hvalname} to scalar.") + updated_hvals[hvalname] = value[0] + hvals.update(updated_hvals) + return hvals def _get_uvals(self): From f4378bd78a86b6f6cb3c995c247a298a3470c61f Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 12 Apr 2023 11:03:52 -0500 Subject: [PATCH 2/7] simplify, add test --- nose_ignores.txt | 1 + tests/tests.yaml | 1 + yt/frontends/gadget/data_structures.py | 10 ++--- .../gadget/tests/test_gadget_pytests.py | 43 +++++++++++++++++++ 4 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 yt/frontends/gadget/tests/test_gadget_pytests.py diff --git a/nose_ignores.txt b/nose_ignores.txt index 96b89bc4f93..30db59c9879 100644 --- a/nose_ignores.txt +++ b/nose_ignores.txt @@ -25,3 +25,4 @@ --ignore-file=test_stream_particles\.py --ignore-file=test_stream_stretched\.py --ignore-file=test_version\.py +--ignore-file=test_gadget_pytests\.py diff --git a/tests/tests.yaml b/tests/tests.yaml index 5ee2b726e9e..42c445e102c 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -235,6 +235,7 @@ other_tests: - "--ignore-file=test_stream_particles\\.py" - "--ignore-file=test_stream_stretched\\.py" - "--ignore-file=test_version\\.py" + - "--ignore-file=yt.frontends.gadget.tests.test_gadget_pytests\\.py" - "--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF" - "--exclude-test=yt.frontends.adaptahop.tests.test_outputs" - "--exclude-test=yt.frontends.stream.tests.test_stream_particles.test_stream_non_cartesian_particles" diff --git a/yt/frontends/gadget/data_structures.py b/yt/frontends/gadget/data_structures.py index 2ed1449f25a..98f56c7ffb1 100644 --- a/yt/frontends/gadget/data_structures.py +++ b/yt/frontends/gadget/data_structures.py @@ -633,13 +633,9 @@ def _get_hvals(self): # ensure that 1-element arrays are reduced to scalars updated_hvals = {} for hvalname, value in hvals.items(): - if isinstance(value, np.ndarray): - if value.ndim == 0: - mylog.warning(f"Casting 0d {hvalname} to float.") - updated_hvals[hvalname] = float(value) - elif value.ndim == 1 and value.size == 1: - mylog.warning(f"Reducing length-1 array {hvalname} to scalar.") - updated_hvals[hvalname] = value[0] + if isinstance(value, np.ndarray) and value.size == 1: + mylog.info(f"Reducing singleton array {hvalname} to scalar.") + updated_hvals[hvalname] = value.item() hvals.update(updated_hvals) return hvals diff --git a/yt/frontends/gadget/tests/test_gadget_pytests.py b/yt/frontends/gadget/tests/test_gadget_pytests.py new file mode 100644 index 00000000000..07d8c76360c --- /dev/null +++ b/yt/frontends/gadget/tests/test_gadget_pytests.py @@ -0,0 +1,43 @@ +import numpy as np + +import yt +from yt.testing import requires_file, requires_module +from yt.utilities.on_demand_imports import _h5py as h5py + + +@requires_file("snapshot_033/snap_033.0.hdf5") +@requires_module("h5py") +def test_gadget_header_array_reduction(tmp_path): + # first get a real header + ds = yt.load("snapshot_033/snap_033.0.hdf5") + hvals = ds._get_hvals() + hvals_orig = hvals.copy() + # wrap some of the values in nested arrays + hvals["Redshift"] = np.array( + [ + hvals["Redshift"], + ] + ) + hvals["Omega0"] = np.array( + [ + [ + hvals["Omega0"], + ], + ] + ) + + # drop those header values into a fake header-only file + tmp_snpshot_dir = tmp_path / "snapshot_033" + tmp_snpshot_dir.mkdir() + tmp_header_only_file = str(tmp_snpshot_dir / "fake_gadget_header.hdf5") + with h5py.File(tmp_header_only_file, mode="w") as f: + headergrp = f.create_group("Header") + for field, val in hvals.items(): + headergrp.attrs[field] = val + + # trick the dataset into using the header file and make sure the + # arrays are reduced + ds._input_filename = tmp_header_only_file + hvals = ds._get_hvals() + assert hvals["Redshift"] == hvals_orig["Redshift"] + assert hvals["Omega0"] == hvals_orig["Omega0"] From 767697a71d299c27efd7db502f6447f621b7ec86 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 12 Apr 2023 11:11:57 -0500 Subject: [PATCH 3/7] adjust test --- yt/frontends/gadget/tests/test_gadget_pytests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/yt/frontends/gadget/tests/test_gadget_pytests.py b/yt/frontends/gadget/tests/test_gadget_pytests.py index 07d8c76360c..bc829d33bf2 100644 --- a/yt/frontends/gadget/tests/test_gadget_pytests.py +++ b/yt/frontends/gadget/tests/test_gadget_pytests.py @@ -12,7 +12,7 @@ def test_gadget_header_array_reduction(tmp_path): ds = yt.load("snapshot_033/snap_033.0.hdf5") hvals = ds._get_hvals() hvals_orig = hvals.copy() - # wrap some of the values in nested arrays + # wrap some of the scalar values in nested arrays hvals["Redshift"] = np.array( [ hvals["Redshift"], @@ -39,5 +39,6 @@ def test_gadget_header_array_reduction(tmp_path): # arrays are reduced ds._input_filename = tmp_header_only_file hvals = ds._get_hvals() - assert hvals["Redshift"] == hvals_orig["Redshift"] - assert hvals["Omega0"] == hvals_orig["Omega0"] + for attr in ("Redshift", "Omega0"): + assert hvals[attr] == hvals_orig[attr] + assert isinstance(hvals[attr], np.ndarray) is False From ae1bb6bc05ce8fc6f8fb4316dd48cf893b9fa114 Mon Sep 17 00:00:00 2001 From: chrishavlin Date: Wed, 12 Apr 2023 12:17:55 -0500 Subject: [PATCH 4/7] correctly ignore-file... --- tests/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests.yaml b/tests/tests.yaml index 42c445e102c..b94922bc1da 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -235,7 +235,7 @@ other_tests: - "--ignore-file=test_stream_particles\\.py" - "--ignore-file=test_stream_stretched\\.py" - "--ignore-file=test_version\\.py" - - "--ignore-file=yt.frontends.gadget.tests.test_gadget_pytests\\.py" + - "--ignore-file=test_gadget_pytests\\.py" - "--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF" - "--exclude-test=yt.frontends.adaptahop.tests.test_outputs" - "--exclude-test=yt.frontends.stream.tests.test_stream_particles.test_stream_non_cartesian_particles" From ced608ee587d554537140b29df3fc32762464578 Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Wed, 14 Jun 2023 10:14:47 -0500 Subject: [PATCH 5/7] Update yt/frontends/gadget/data_structures.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/frontends/gadget/data_structures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yt/frontends/gadget/data_structures.py b/yt/frontends/gadget/data_structures.py index 98f56c7ffb1..37960ba2c62 100644 --- a/yt/frontends/gadget/data_structures.py +++ b/yt/frontends/gadget/data_structures.py @@ -634,7 +634,7 @@ def _get_hvals(self): updated_hvals = {} for hvalname, value in hvals.items(): if isinstance(value, np.ndarray) and value.size == 1: - mylog.info(f"Reducing singleton array {hvalname} to scalar.") + mylog.info("Reducing single-element array %s to scalar.", hvalname) updated_hvals[hvalname] = value.item() hvals.update(updated_hvals) From 9b47d076b6dec15d67bf6d725d0b96730a7688df Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Wed, 14 Jun 2023 10:58:52 -0500 Subject: [PATCH 6/7] rename test file --- nose_ignores.txt | 2 +- tests/tests.yaml | 2 +- .../tests/{test_gadget_pytests.py => test_gadget_pytest.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename yt/frontends/gadget/tests/{test_gadget_pytests.py => test_gadget_pytest.py} (100%) diff --git a/nose_ignores.txt b/nose_ignores.txt index 153b641768e..fb2372f5331 100644 --- a/nose_ignores.txt +++ b/nose_ignores.txt @@ -32,5 +32,5 @@ --ignore-file=test_stream_particles\.py --ignore-file=test_stream_stretched\.py --ignore-file=test_version\.py ---ignore-file=test_gadget_pytests\.py +--ignore-file=test_gadget_pytest\.py --ignore-file=test_vr_orientation\.py diff --git a/tests/tests.yaml b/tests/tests.yaml index fd79a051517..7d0168a5373 100644 --- a/tests/tests.yaml +++ b/tests/tests.yaml @@ -204,7 +204,7 @@ other_tests: - "--ignore-file=test_stream_particles\\.py" - "--ignore-file=test_stream_stretched\\.py" - "--ignore-file=test_version\\.py" - - "--ignore-file=test_gadget_pytests\\.py" + - "--ignore-file=test_gadget_pytest\\.py" - "--ignore-file=test_vr_orientation\\.py" - "--exclude-test=yt.frontends.gdf.tests.test_outputs.TestGDF" - "--exclude-test=yt.frontends.adaptahop.tests.test_outputs" diff --git a/yt/frontends/gadget/tests/test_gadget_pytests.py b/yt/frontends/gadget/tests/test_gadget_pytest.py similarity index 100% rename from yt/frontends/gadget/tests/test_gadget_pytests.py rename to yt/frontends/gadget/tests/test_gadget_pytest.py From 10b8a8b6f1ad9befad167dcae0cd6f008a290e70 Mon Sep 17 00:00:00 2001 From: Chris Havlin Date: Tue, 20 Jun 2023 10:04:16 -0500 Subject: [PATCH 7/7] Update yt/frontends/gadget/tests/test_gadget_pytest.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Robert --- yt/frontends/gadget/tests/test_gadget_pytest.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/yt/frontends/gadget/tests/test_gadget_pytest.py b/yt/frontends/gadget/tests/test_gadget_pytest.py index bc829d33bf2..b493b4a97ea 100644 --- a/yt/frontends/gadget/tests/test_gadget_pytest.py +++ b/yt/frontends/gadget/tests/test_gadget_pytest.py @@ -13,18 +13,8 @@ def test_gadget_header_array_reduction(tmp_path): hvals = ds._get_hvals() hvals_orig = hvals.copy() # wrap some of the scalar values in nested arrays - hvals["Redshift"] = np.array( - [ - hvals["Redshift"], - ] - ) - hvals["Omega0"] = np.array( - [ - [ - hvals["Omega0"], - ], - ] - ) + hvals["Redshift"] = np.array([hvals["Redshift"]]) + hvals["Omega0"] = np.array([[hvals["Omega0"]]]) # drop those header values into a fake header-only file tmp_snpshot_dir = tmp_path / "snapshot_033"