From 8610805a9ebd14c1e8fd770db353f80a2e072115 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Wed, 5 Jun 2024 03:28:52 +0530 Subject: [PATCH 1/7] Add validation for schema and manifests --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 763e86394..34ebaf8bd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -53,3 +53,4 @@ filterwarnings = ignore:Animating a NDCube does not support transposing the array. The world axes may not display as expected because the array will not be transposed:UserWarning # This is raised by the Windows and mac os build for visualization.rst ignore:FigureCanvasAgg is non-interactive, and thus cannot be shown:UserWarning + From 149944f021618cb58243b4bb842db62f796db898 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Thu, 4 Jul 2024 19:13:09 +0530 Subject: [PATCH 2/7] revert small change --- pytest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 34ebaf8bd..763e86394 100644 --- a/pytest.ini +++ b/pytest.ini @@ -53,4 +53,3 @@ filterwarnings = ignore:Animating a NDCube does not support transposing the array. The world axes may not display as expected because the array will not be transposed:UserWarning # This is raised by the Windows and mac os build for visualization.rst ignore:FigureCanvasAgg is non-interactive, and thus cannot be shown:UserWarning - From 98e633ca2fd664df1570a08475d56c0534ad20a4 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 5 Aug 2024 01:55:41 +0530 Subject: [PATCH 3/7] Add asdf as an optional dep --- pyproject.toml | 9 +++++++-- tox.ini | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8756cc1fd..754bd98ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,11 +54,16 @@ plotting = [ reproject = [ "reproject>=0.7.1", ] +asdf = [ + "asdf>=2.14.4", + "astropy>=5.3.0", + "gwcs>=0.20" +] all = [ - "ndcube[plotting,reproject]", + "ndcube[plotting,reproject,asdf]", ] dev = [ - "ndcube[tests,docs,plotting,reproject]", + "ndcube[tests,docs,plotting,reproject,asdf]", ] [project.urls] repository = "https://docs.sunpy.org/projects/ndcube" diff --git a/tox.ini b/tox.ini index 2655a0b5d..24e66e651 100644 --- a/tox.ini +++ b/tox.ini @@ -63,6 +63,7 @@ deps = figure-!devdeps: scipy # The following indicates which extras_require will be installed extras = + asdf plotting reproject tests From 789e841bd7491688b405510a82c2fa05be1eedfe Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Mon, 5 Aug 2024 12:19:08 +0530 Subject: [PATCH 4/7] Remove asdf as an optional dependency --- pyproject.toml | 9 ++------- tox.ini | 1 - 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 754bd98ea..8756cc1fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,16 +54,11 @@ plotting = [ reproject = [ "reproject>=0.7.1", ] -asdf = [ - "asdf>=2.14.4", - "astropy>=5.3.0", - "gwcs>=0.20" -] all = [ - "ndcube[plotting,reproject,asdf]", + "ndcube[plotting,reproject]", ] dev = [ - "ndcube[tests,docs,plotting,reproject,asdf]", + "ndcube[tests,docs,plotting,reproject]", ] [project.urls] repository = "https://docs.sunpy.org/projects/ndcube" diff --git a/tox.ini b/tox.ini index 24e66e651..2655a0b5d 100644 --- a/tox.ini +++ b/tox.ini @@ -63,7 +63,6 @@ deps = figure-!devdeps: scipy # The following indicates which extras_require will be installed extras = - asdf plotting reproject tests From 24af793cc51c0cff8569710b8887655ea699fe12 Mon Sep 17 00:00:00 2001 From: ViciousEagle03 Date: Sun, 25 Aug 2024 05:13:30 +0530 Subject: [PATCH 5/7] Add serialization logic for the NDCubeSequence and NDCollection --- .../asdf/converters/ndcollection_converter.py | 30 +++++++++++ .../converters/ndcubesequence_converter.py | 25 +++++++++ .../tests/test_ndcollection_converter.py | 51 +++++++++++++++++++ .../tests/test_ndcubesequence_converter.py | 20 ++++++++ ndcube/asdf/entry_points.py | 5 +- .../resources/manifests/ndcube-0.1.0.yaml | 6 +++ .../resources/schemas/ndcollection-0.1.0.yaml | 30 +++++++++++ .../schemas/ndcube_sequence-0.1.0.yaml | 25 +++++++++ 8 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 ndcube/asdf/converters/ndcollection_converter.py create mode 100644 ndcube/asdf/converters/ndcubesequence_converter.py create mode 100644 ndcube/asdf/converters/tests/test_ndcollection_converter.py create mode 100644 ndcube/asdf/converters/tests/test_ndcubesequence_converter.py create mode 100644 ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml create mode 100644 ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py new file mode 100644 index 000000000..921365539 --- /dev/null +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -0,0 +1,30 @@ +from asdf.extension import Converter + + +class NDCollectionConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcube/ndcollection-*"] + types = ["ndcube.ndcollection.NDCollection"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.ndcollection import NDCollection + + key_value_pairs = list(zip(node["keys"], node["value"])) + aligned_axes = list(node.get("aligned_axes").values()) + aligned_axes = tuple(tuple(lst) for lst in aligned_axes) + ndcollection = NDCollection(key_value_pairs, + meta=node.get("meta"), + aligned_axes = aligned_axes) + return ndcollection + + + def to_yaml_tree(self, ndcollection, tag, ctx): + + node = {} + node["keys"] = tuple(ndcollection.keys()) + node["value"] = tuple(ndcollection.values()) + if ndcollection.meta is not None: + node["meta"] = ndcollection.meta + if ndcollection._aligned_axes is not None: + node["aligned_axes"] = ndcollection._aligned_axes + + return node diff --git a/ndcube/asdf/converters/ndcubesequence_converter.py b/ndcube/asdf/converters/ndcubesequence_converter.py new file mode 100644 index 000000000..ba73e45cf --- /dev/null +++ b/ndcube/asdf/converters/ndcubesequence_converter.py @@ -0,0 +1,25 @@ +from asdf.extension import Converter + + +class NDCubeSequenceConverter(Converter): + tags = ["tag:sunpy.org:ndcube/ndcube/ndcube_sequence-*"] + types = ["ndcube.ndcube_sequence.NDCubeSequence"] + + def from_yaml_tree(self, node, tag, ctx): + from ndcube.ndcube_sequence import NDCubeSequence + + return NDCubeSequence(node["data"], + meta=node.get("meta"), + common_axis=node.get("common_axis")) + + + def to_yaml_tree(self, ndcseq, tag, ctx): + + node = {} + node["data"] = ndcseq.data + if ndcseq.meta is not None: + node["meta"] = ndcseq.meta + if ndcseq._common_axis is not None: + node["common_axis"] = ndcseq._common_axis + + return node diff --git a/ndcube/asdf/converters/tests/test_ndcollection_converter.py b/ndcube/asdf/converters/tests/test_ndcollection_converter.py new file mode 100644 index 000000000..3dff3bf28 --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcollection_converter.py @@ -0,0 +1,51 @@ +import pytest +from gwcs import __version__ as gwcs_version +from packaging.version import Version + +import asdf + +from ndcube.ndcollection import NDCollection +from ndcube.ndcube_sequence import NDCubeSequence +from ndcube.tests.helpers import assert_collections_equal + + +@pytest.fixture +def create_ndcollection_cube(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim): + aligned_axes = ((1, 2), (1, 2), (1, 2)) + cube_collection = NDCollection([("cube0", ndcube_gwcs_3d_ln_lt_l), + ("cube1", ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc), + ("cube2", ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim)], + aligned_axes=aligned_axes) + + return cube_collection + + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization_cube(create_ndcollection_cube, tmp_path): + ndcollection = create_ndcollection_cube + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcollection + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_collections_equal(af["ndcube_gwcs"], ndcollection) +@pytest.fixture +def create_ndcollection_sequence(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim): + + sequence02 = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim]) + sequence20 = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim, ndcube_gwcs_3d_ln_lt_l]) + seq_collection = NDCollection([("seq0", sequence02), ("seq1", sequence20)], aligned_axes="all") + return seq_collection + + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization_sequence(create_ndcollection_sequence, tmp_path): + ndcollection = create_ndcollection_sequence + file_path = tmp_path / "test.asdf" + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcollection + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_collections_equal(af["ndcube_gwcs"], ndcollection) diff --git a/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py b/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py new file mode 100644 index 000000000..bf2faaa4a --- /dev/null +++ b/ndcube/asdf/converters/tests/test_ndcubesequence_converter.py @@ -0,0 +1,20 @@ +import pytest +from gwcs import __version__ as gwcs_version +from packaging.version import Version + +import asdf + +from ndcube.ndcube_sequence import NDCubeSequence +from ndcube.tests.helpers import assert_cubesequences_equal + + +@pytest.mark.skipif(Version(gwcs_version) < Version("0.20"), reason="Requires gwcs>=0.20") +def test_serialization(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc, tmp_path): + file_path = tmp_path / "test.asdf" + ndcseq = NDCubeSequence([ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc], common_axis=1) + with asdf.AsdfFile() as af: + af["ndcube_gwcs"] = ndcseq + af.write_to(file_path) + + with asdf.open(file_path) as af: + assert_cubesequences_equal(af["ndcube_gwcs"], ndcseq) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 0919b45f7..973e49ba6 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -34,15 +34,16 @@ def get_extensions(): from ndcube.asdf.converters.compoundwcs_converter import CompoundConverter from ndcube.asdf.converters.extracoords_converter import ExtraCoordsConverter from ndcube.asdf.converters.globalcoords_converter import GlobalCoordsConverter + from ndcube.asdf.converters.ndcollection_converter import NDCollectionConverter from ndcube.asdf.converters.ndcube_converter import NDCubeConverter from ndcube.asdf.converters.reorderedwcs_converter import ReorderedConverter from ndcube.asdf.converters.resampled_converter import ResampledConverter + from ndcube.asdf.converters.ndcubesequence_converter import NDCubeSequenceConverter from ndcube.asdf.converters.tablecoord_converter import ( QuantityTableCoordinateConverter, SkyCoordTableCoordinateConverter, TimeTableCoordConverter, ) - ndcube_converters = [ NDCubeConverter(), ExtraCoordsConverter(), @@ -53,6 +54,8 @@ def get_extensions(): ResampledConverter(), ReorderedConverter(), CompoundConverter(), + NDCubeSequenceConverter(), + NDCollectionConverter(), ] _manifest_uri = "asdf://sunpy.org/ndcube/manifests/ndcube-0.1.0" diff --git a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml index d9e3a02bf..bcb0b8a76 100644 --- a/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml +++ b/ndcube/asdf/resources/manifests/ndcube-0.1.0.yaml @@ -27,8 +27,14 @@ tags: - tag_uri: "tag:sunpy.org:ndcube/resampledwcs-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/resampledwcs-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-0.1.0" + - tag_uri: "tag:sunpy.org:ndcube/reorderedwcs-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/reorderedwcs-0.1.0" - tag_uri: "tag:sunpy.org:ndcube/compoundwcs-0.1.0" schema_uri: "asdf://sunpy.org/ndcube/schemas/compoundwcs-0.1.0" + + - tag_uri: "tag:sunpy.org:ndcube/ndcube/ndcollection-0.1.0" + schema_uri: "asdf://sunpy.org/ndcube/schemas/ndcollection-0.1.0" diff --git a/ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml new file mode 100644 index 000000000..02c76e418 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndcollection-0.1.0.yaml @@ -0,0 +1,30 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndcollection-0.1.0" + +title: + Represents the ndcube.ndcollection.NDCollection object + +description: + Represents the ndcube ndcube.ndcollection.NDCollection object + +type: object +properties: + keys: + type: array + value: + type: array + items: + - type: object + oneOf: + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" + - tag: "tag:sunpy.org:ndcube/ndcube/ndcube_sequence-0.*" + aligned_axes: + anyOf: + - type: object + - type: string + +required: [keys, value] +additionalProperties: true +... diff --git a/ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml b/ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml new file mode 100644 index 000000000..155665502 --- /dev/null +++ b/ndcube/asdf/resources/schemas/ndcube_sequence-0.1.0.yaml @@ -0,0 +1,25 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://sunpy.org/ndcube/schemas/ndcube_sequence-0.1.0" + +title: + Represents the ndcube.ndcube_sequence.NDCubeSequence object + +description: + Represents the ndcube.ndcube_sequence.NDCubeSequence object + +type: object +properties: + data: + type: array + items: + tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*" + meta: + type: object + common_axis: + type: integer + +required: [data] +additionalProperties: true +... From 494e2c565e10bebc30f7764cd0840613ed35c8c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:52:08 +0000 Subject: [PATCH 6/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ndcube/asdf/entry_points.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndcube/asdf/entry_points.py b/ndcube/asdf/entry_points.py index 973e49ba6..8fd907977 100644 --- a/ndcube/asdf/entry_points.py +++ b/ndcube/asdf/entry_points.py @@ -36,9 +36,9 @@ def get_extensions(): from ndcube.asdf.converters.globalcoords_converter import GlobalCoordsConverter from ndcube.asdf.converters.ndcollection_converter import NDCollectionConverter from ndcube.asdf.converters.ndcube_converter import NDCubeConverter + from ndcube.asdf.converters.ndcubesequence_converter import NDCubeSequenceConverter from ndcube.asdf.converters.reorderedwcs_converter import ReorderedConverter from ndcube.asdf.converters.resampled_converter import ResampledConverter - from ndcube.asdf.converters.ndcubesequence_converter import NDCubeSequenceConverter from ndcube.asdf.converters.tablecoord_converter import ( QuantityTableCoordinateConverter, SkyCoordTableCoordinateConverter, From db8f22d609b54be59c2722a647dcf5041ade7856 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 6 Nov 2024 09:55:58 +0000 Subject: [PATCH 7/7] Minor tweaks --- ndcube/asdf/converters/ndcollection_converter.py | 6 ++---- ndcube/asdf/converters/ndcubesequence_converter.py | 2 -- ndcube/asdf/converters/tests/test_ndcollection_converter.py | 2 ++ 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ndcube/asdf/converters/ndcollection_converter.py b/ndcube/asdf/converters/ndcollection_converter.py index 921365539..810bb151c 100644 --- a/ndcube/asdf/converters/ndcollection_converter.py +++ b/ndcube/asdf/converters/ndcollection_converter.py @@ -12,13 +12,11 @@ def from_yaml_tree(self, node, tag, ctx): aligned_axes = list(node.get("aligned_axes").values()) aligned_axes = tuple(tuple(lst) for lst in aligned_axes) ndcollection = NDCollection(key_value_pairs, - meta=node.get("meta"), - aligned_axes = aligned_axes) + meta=node.get("meta"), + aligned_axes = aligned_axes) return ndcollection - def to_yaml_tree(self, ndcollection, tag, ctx): - node = {} node["keys"] = tuple(ndcollection.keys()) node["value"] = tuple(ndcollection.values()) diff --git a/ndcube/asdf/converters/ndcubesequence_converter.py b/ndcube/asdf/converters/ndcubesequence_converter.py index ba73e45cf..419c69497 100644 --- a/ndcube/asdf/converters/ndcubesequence_converter.py +++ b/ndcube/asdf/converters/ndcubesequence_converter.py @@ -12,9 +12,7 @@ def from_yaml_tree(self, node, tag, ctx): meta=node.get("meta"), common_axis=node.get("common_axis")) - def to_yaml_tree(self, ndcseq, tag, ctx): - node = {} node["data"] = ndcseq.data if ndcseq.meta is not None: diff --git a/ndcube/asdf/converters/tests/test_ndcollection_converter.py b/ndcube/asdf/converters/tests/test_ndcollection_converter.py index 3dff3bf28..19158da4f 100644 --- a/ndcube/asdf/converters/tests/test_ndcollection_converter.py +++ b/ndcube/asdf/converters/tests/test_ndcollection_converter.py @@ -30,6 +30,8 @@ def test_serialization_cube(create_ndcollection_cube, tmp_path): with asdf.open(file_path) as af: assert_collections_equal(af["ndcube_gwcs"], ndcollection) + + @pytest.fixture def create_ndcollection_sequence(ndcube_gwcs_3d_ln_lt_l, ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim):