From a878635361f3ae71f5025be791be08d499c8f610 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 20 Feb 2024 15:10:33 +0800 Subject: [PATCH 1/7] clib.Session: Add the virtualfile_to_data method for creating virtual files for output --- doc/api/index.rst | 1 + pygmt/clib/session.py | 70 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/doc/api/index.rst b/doc/api/index.rst index 60a1240150e..c460beb4aaf 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -293,6 +293,7 @@ conversion of Python variables to GMT virtual files: clib.Session.virtualfile_from_matrix clib.Session.virtualfile_from_vectors clib.Session.virtualfile_from_grid + clib.Session.virtualfile_to_data Low level access (these are mostly used by the :mod:`pygmt.clib` package): diff --git a/pygmt/clib/session.py b/pygmt/clib/session.py index 3aa42c885df..e30b3fd703b 100644 --- a/pygmt/clib/session.py +++ b/pygmt/clib/session.py @@ -1609,6 +1609,76 @@ def virtualfile_from_data( # noqa: PLR0912 return file_context + @contextlib.contextmanager + def virtualfile_to_data( + self, kind: Literal["dataset", "grid"] = "dataset", fname: str | None = None + ): + """ + Create a virtual file for storing output data in a data container or yield the + actual file name. + + Parameters + ---------- + kind + The kind of data container to create. Valid values are ``"dataset"`` and + ``"grid"``. Ignored if ``fname`` is specified. + fname + If given, yield the actual file name instead of the virtual file name. + + Yields + ------ + vfile : str + Name of the virtual file or the output file name. + + Examples + -------- + >>> from pathlib import Path + >>> from pygmt.clib import Session + >>> from pygmt.datatypes import _GMT_DATASET, _GMT_GRID + >>> from pygmt.helpers import GMTTempFile + >>> + >>> # Create a virtual file for storing the output table. + >>> with GMTTempFile(suffix=".txt") as tmpfile: + ... with open(tmpfile.name, mode="w") as fp: + ... print("1.0 2.0 3.0 TEXT", file=fp) + ... with Session() as lib: + ... with lib.virtualfile_to_data(kind="dataset") as vouttbl: + ... lib.call_module("read", f"{tmpfile.name} {vouttbl} -Td") + ... ds = lib.read_virtualfile(vouttbl, kind="dataset") + >>> isinstance(ds.contents, _GMT_DATASET) + True + >>> + >>> # Create a virtual file for storing the output grid. + >>> with Session() as lib: + ... with lib.virtualfile_to_data(kind="grid") as voutgrd: + ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") + ... outgrd = lib.read_virtualfile(voutgrd, kind="grid") + >>> isinstance(outgrd.contents, _GMT_GRID) + True + >>> + >>> # Write data to file without creating a virtual file + >>> with GMTTempFile(suffix=".nc") as tmpfile: + ... with Session() as lib: + ... with lib.virtualfile_to_data( + ... kind="grid", fname=tmpfile.name + ... ) as voutgrd: + ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") + ... assert voutgrd == tmpfile.name + ... assert Path(voutgrd).stat().st_size > 0 + """ + # If fname is given, yield the output file name. + if fname is not None: + yield fname + # Otherwise, create a virtual file for storing the output data. + else: + # Determine the family and geometry from kind + family, geometry = { + "grid": ("GMT_IS_GRID", "GMT_IS_SURFACE"), + "dataset": ("GMT_IS_DATASET", "GMT_IS_PLP"), + }[kind] + with self.open_virtualfile(family, geometry, "GMT_OUT", None) as vfile: + yield vfile + def read_virtualfile( self, vfname: str, kind: Literal["dataset", "grid", None] = None ): From a633abf2d012378b7f6f6a6a82f3795022eaf7e9 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Mon, 26 Feb 2024 17:00:28 +0800 Subject: [PATCH 2/7] Rename virtualfile_to_data to virtualfile_out --- doc/api/index.rst | 2 +- pygmt/clib/session.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/api/index.rst b/doc/api/index.rst index c460beb4aaf..0349fae1b3e 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -293,7 +293,7 @@ conversion of Python variables to GMT virtual files: clib.Session.virtualfile_from_matrix clib.Session.virtualfile_from_vectors clib.Session.virtualfile_from_grid - clib.Session.virtualfile_to_data + clib.Session.virtualfile_out Low level access (these are mostly used by the :mod:`pygmt.clib` package): diff --git a/pygmt/clib/session.py b/pygmt/clib/session.py index e30b3fd703b..af91d6b1285 100644 --- a/pygmt/clib/session.py +++ b/pygmt/clib/session.py @@ -1610,7 +1610,7 @@ def virtualfile_from_data( # noqa: PLR0912 return file_context @contextlib.contextmanager - def virtualfile_to_data( + def virtualfile_out( self, kind: Literal["dataset", "grid"] = "dataset", fname: str | None = None ): """ @@ -1642,7 +1642,7 @@ def virtualfile_to_data( ... with open(tmpfile.name, mode="w") as fp: ... print("1.0 2.0 3.0 TEXT", file=fp) ... with Session() as lib: - ... with lib.virtualfile_to_data(kind="dataset") as vouttbl: + ... with lib.virtualfile_out(kind="dataset") as vouttbl: ... lib.call_module("read", f"{tmpfile.name} {vouttbl} -Td") ... ds = lib.read_virtualfile(vouttbl, kind="dataset") >>> isinstance(ds.contents, _GMT_DATASET) @@ -1650,7 +1650,7 @@ def virtualfile_to_data( >>> >>> # Create a virtual file for storing the output grid. >>> with Session() as lib: - ... with lib.virtualfile_to_data(kind="grid") as voutgrd: + ... with lib.virtualfile_out(kind="grid") as voutgrd: ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") ... outgrd = lib.read_virtualfile(voutgrd, kind="grid") >>> isinstance(outgrd.contents, _GMT_GRID) @@ -1659,7 +1659,7 @@ def virtualfile_to_data( >>> # Write data to file without creating a virtual file >>> with GMTTempFile(suffix=".nc") as tmpfile: ... with Session() as lib: - ... with lib.virtualfile_to_data( + ... with lib.virtualfile_out( ... kind="grid", fname=tmpfile.name ... ) as voutgrd: ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") From 59527e3dcdc1edfb3c7ce2bc7b2cdc49d2e2d11d Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 27 Feb 2024 19:59:51 +0800 Subject: [PATCH 3/7] Improve the docstrings --- pygmt/clib/session.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pygmt/clib/session.py b/pygmt/clib/session.py index af91d6b1285..05dda0d8950 100644 --- a/pygmt/clib/session.py +++ b/pygmt/clib/session.py @@ -1614,21 +1614,26 @@ def virtualfile_out( self, kind: Literal["dataset", "grid"] = "dataset", fname: str | None = None ): """ - Create a virtual file for storing output data in a data container or yield the - actual file name. + Create a virtual file or an actual file for storing output data. + + If ``fname`` is not given, a virtual file will be created to store the output + data into a GMT data container and the function yields the name of the virutal + file. Otherwise, the output data will be written into the specified file and the + function simply yields the actual file name. Parameters ---------- kind - The kind of data container to create. Valid values are ``"dataset"`` and - ``"grid"``. Ignored if ``fname`` is specified. + The data kind of the virtual file to create. Valid values are ``"dataset"`` + and ``"grid"``. Ignored if ``fname`` is specified. fname - If given, yield the actual file name instead of the virtual file name. + The name of the actual file to write the output data. No virtual file will + be created. Yields ------ vfile : str - Name of the virtual file or the output file name. + Name of the virtual file or the actual file. Examples -------- @@ -1666,11 +1671,9 @@ def virtualfile_out( ... assert voutgrd == tmpfile.name ... assert Path(voutgrd).stat().st_size > 0 """ - # If fname is given, yield the output file name. - if fname is not None: + if fname is not None: # Yield the actual file name. yield fname - # Otherwise, create a virtual file for storing the output data. - else: + else: # Create a virtual file for storing the output data. # Determine the family and geometry from kind family, geometry = { "grid": ("GMT_IS_GRID", "GMT_IS_SURFACE"), From 9108917d5cd73bbc8897937636e8a3252ff7e50f Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 28 Feb 2024 10:51:41 +0800 Subject: [PATCH 4/7] Remove some doctests from virtualfile_out and improve the read_virtualfile doctests --- pygmt/clib/session.py | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/pygmt/clib/session.py b/pygmt/clib/session.py index 05dda0d8950..3ad2ec8ab48 100644 --- a/pygmt/clib/session.py +++ b/pygmt/clib/session.py @@ -1639,7 +1639,7 @@ def virtualfile_out( -------- >>> from pathlib import Path >>> from pygmt.clib import Session - >>> from pygmt.datatypes import _GMT_DATASET, _GMT_GRID + >>> from pygmt.datatypes import _GMT_DATASET >>> from pygmt.helpers import GMTTempFile >>> >>> # Create a virtual file for storing the output table. @@ -1652,24 +1652,13 @@ def virtualfile_out( ... ds = lib.read_virtualfile(vouttbl, kind="dataset") >>> isinstance(ds.contents, _GMT_DATASET) True - >>> - >>> # Create a virtual file for storing the output grid. - >>> with Session() as lib: - ... with lib.virtualfile_out(kind="grid") as voutgrd: - ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") - ... outgrd = lib.read_virtualfile(voutgrd, kind="grid") - >>> isinstance(outgrd.contents, _GMT_GRID) - True - >>> - >>> # Write data to file without creating a virtual file + >>> # Write data to an actual file without creating a virtual file. >>> with GMTTempFile(suffix=".nc") as tmpfile: ... with Session() as lib: - ... with lib.virtualfile_out( - ... kind="grid", fname=tmpfile.name - ... ) as voutgrd: - ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") + ... with lib.virtualfile_out(fname=tmpfile.name) as voutgrd: ... assert voutgrd == tmpfile.name - ... assert Path(voutgrd).stat().st_size > 0 + ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") + ... assert Path(voutgrd).stat().st_size > 0 """ if fname is not None: # Yield the actual file name. yield fname @@ -1706,27 +1695,23 @@ def read_virtualfile( ... with GMTTempFile(suffix=".txt") as tmpfile: ... with open(tmpfile.name, mode="w") as fp: ... print("1.0 2.0 3.0 TEXT", file=fp) - ... with lib.open_virtualfile( - ... "GMT_IS_DATASET", "GMT_IS_PLP", "GMT_OUT", None - ... ) as vfile: - ... lib.call_module("read", f"{tmpfile.name} {vfile} -Td") + ... with lib.virtualfile_out(kind="dataset") as vouttbl: + ... lib.call_module("read", f"{tmpfile.name} {vouttbl} -Td") ... # Read the virtual file as a void pointer - ... void_pointer = lib.read_virtualfile(vfile) + ... void_pointer = lib.read_virtualfile(vouttbl) ... assert isinstance(void_pointer, int) # void pointer is an int ... # Read the virtual file as a dataset - ... data_pointer = lib.read_virtualfile(vfile, kind="dataset") + ... data_pointer = lib.read_virtualfile(vouttbl, kind="dataset") ... assert isinstance(data_pointer, ctp.POINTER(_GMT_DATASET)) >>> >>> # Read grid from a virtual file >>> with Session() as lib: - ... with lib.open_virtualfile( - ... "GMT_IS_GRID", "GMT_IS_SURFACE", "GMT_OUT", None - ... ) as vfile: - ... lib.call_module("read", f"@earth_relief_01d_g {vfile} -Tg") + ... with lib.virtualfile_out(kind="grid") as voutgrd: + ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") ... # Read the virtual file as a void pointer - ... void_pointer = lib.read_virtualfile(vfile) + ... void_pointer = lib.read_virtualfile(voutgrd) ... assert isinstance(void_pointer, int) # void pointer is an int - ... data_pointer = lib.read_virtualfile(vfile, kind="grid") + ... data_pointer = lib.read_virtualfile(voutgrd, kind="grid") ... assert isinstance(data_pointer, ctp.POINTER(_GMT_GRID)) Returns From ee4cadad755102d56882c38f306ab426157e2921 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 28 Feb 2024 10:54:41 +0800 Subject: [PATCH 5/7] Sort alphabetically --- pygmt/clib/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygmt/clib/session.py b/pygmt/clib/session.py index 3ad2ec8ab48..f69b478a46e 100644 --- a/pygmt/clib/session.py +++ b/pygmt/clib/session.py @@ -1665,8 +1665,8 @@ def virtualfile_out( else: # Create a virtual file for storing the output data. # Determine the family and geometry from kind family, geometry = { - "grid": ("GMT_IS_GRID", "GMT_IS_SURFACE"), "dataset": ("GMT_IS_DATASET", "GMT_IS_PLP"), + "grid": ("GMT_IS_GRID", "GMT_IS_SURFACE"), }[kind] with self.open_virtualfile(family, geometry, "GMT_OUT", None) as vfile: yield vfile From 888f1c9c96b4de496aa2b80368c70fab45ccc1ea Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 28 Feb 2024 12:09:26 +0800 Subject: [PATCH 6/7] Check the content of the output file --- pygmt/clib/session.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pygmt/clib/session.py b/pygmt/clib/session.py index f69b478a46e..8dbda87a284 100644 --- a/pygmt/clib/session.py +++ b/pygmt/clib/session.py @@ -1613,7 +1613,7 @@ def virtualfile_from_data( # noqa: PLR0912 def virtualfile_out( self, kind: Literal["dataset", "grid"] = "dataset", fname: str | None = None ): - """ + r""" Create a virtual file or an actual file for storing output data. If ``fname`` is not given, a virtual file will be created to store the output @@ -1642,23 +1642,25 @@ def virtualfile_out( >>> from pygmt.datatypes import _GMT_DATASET >>> from pygmt.helpers import GMTTempFile >>> - >>> # Create a virtual file for storing the output table. >>> with GMTTempFile(suffix=".txt") as tmpfile: ... with open(tmpfile.name, mode="w") as fp: ... print("1.0 2.0 3.0 TEXT", file=fp) + ... + ... # Create a virtual file for storing the output table. ... with Session() as lib: ... with lib.virtualfile_out(kind="dataset") as vouttbl: ... lib.call_module("read", f"{tmpfile.name} {vouttbl} -Td") ... ds = lib.read_virtualfile(vouttbl, kind="dataset") - >>> isinstance(ds.contents, _GMT_DATASET) - True - >>> # Write data to an actual file without creating a virtual file. - >>> with GMTTempFile(suffix=".nc") as tmpfile: + ... assert isinstance(ds.contents, _GMT_DATASET) + ... + ... # Write data to an actual file without creating a virtual file. ... with Session() as lib: - ... with lib.virtualfile_out(fname=tmpfile.name) as voutgrd: - ... assert voutgrd == tmpfile.name - ... lib.call_module("read", f"@earth_relief_01d_g {voutgrd} -Tg") - ... assert Path(voutgrd).stat().st_size > 0 + ... with lib.virtualfile_out(fname=tmpfile.name) as vouttbl: + ... assert vouttbl == tmpfile.name + ... lib.call_module("read", f"{tmpfile.name} {vouttbl} -Td") + ... with open(vouttbl, mode="r") as fp: + ... line = fp.readline() + ... assert line == "1\t2\t3\tTEXT\n" """ if fname is not None: # Yield the actual file name. yield fname From 519c780ef994c9445861fba3bac5767704a026c6 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Wed, 28 Feb 2024 12:14:38 +0800 Subject: [PATCH 7/7] Remove unused imports in doctests --- pygmt/clib/session.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pygmt/clib/session.py b/pygmt/clib/session.py index 8dbda87a284..652c3c65d9f 100644 --- a/pygmt/clib/session.py +++ b/pygmt/clib/session.py @@ -1637,7 +1637,6 @@ def virtualfile_out( Examples -------- - >>> from pathlib import Path >>> from pygmt.clib import Session >>> from pygmt.datatypes import _GMT_DATASET >>> from pygmt.helpers import GMTTempFile