From 851624ea55ba7af1c6776666b40fa5392476db3e Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Tue, 20 Jun 2023 15:59:08 -0500 Subject: [PATCH 01/10] Added utility copy and info commands. --- src/mdio/commands/utility.py | 167 +++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 src/mdio/commands/utility.py diff --git a/src/mdio/commands/utility.py b/src/mdio/commands/utility.py new file mode 100644 index 00000000..8e19e9c8 --- /dev/null +++ b/src/mdio/commands/utility.py @@ -0,0 +1,167 @@ +"""Dataset CLI Plugin.""" + + +try: + import click + import click_params + + import mdio + import json +except SystemError: + pass + + +DEFAULT_HELP = """ +MDIO CLI utilities. +""" + + +@click.group(help=DEFAULT_HELP) +def cli(): + click.echo(f"MDIO CLI utilities") + + +@cli.command(name="copy") +@click.option( + "-i", + "--input-mdio-path", + required=True, + help="Input mdio path.", + type=click.Path(exists=True), +) +@click.option( + "-o", + "--output-mdio-path", + required=True, + help="Output path or URL to write the mdio dataset.", + type=click.STRING, +) +@click.option( + "-exc", + "--excludes", + required=False, + help="Data to exclude during copy. i.e. chunked_012. The raw data won’t be copied, but it will create an empty array to be filled. If left blank, it will copy everything.", + type=click.STRING, +) +@click.option( + "-inc", + "--includes", + required=False, + help="Data to include during copy. i.e. trace_headers. If this is not specified, and certain data is excluded, it will not copy headers. If you want to preserve headers, specify trace_headers. If left blank, it will copy everything except specified in excludes parameter.", + type=click.STRING, +) +@click.option( + "-storage", + "--storage-options", + required=False, + help="Custom storage options for cloud backends", + type=click_params.JSON, +) +@click.option( + "-overwrite", + "--overwrite", + required=False, + default=False, + help="Flag to overwrite if mdio file if it exists", + type=click.BOOL, + show_default=True, +) +def copy( + input_mdio_path, + output_mdio_path: str, + includes: str = "", + excludes: str = "", + storage_options: dict | None = None, + overwrite: bool = False, +): + """Copy MDIO to MDIO. + Can also copy with empty data to be filled later. See `excludes` + and `includes` parameters. + + More documentation about `excludes` and `includes` can be found + in Zarr's documentation in `zarr.convenience.copy_store`. + + Args: + source: MDIO reader or accessor instance. Data will be copied from here + dest_path_or_buffer: Destination path. Could be any FSSpec mapping. + excludes: Data to exclude during copy. i.e. `chunked_012`. The raw data + won't be copied, but it will create an empty array to be filled. + If left blank, it will copy everything. + includes: Data to include during copy. i.e. `trace_headers`. If this is + not specified, and certain data is excluded, it will not copy headers. + If you want to preserve headers, specify `trace_headers`. If left blank, + it will copy everything except specified in `excludes` parameter. + storage_options: Storage options for the cloud storage backend. + Default is None (will assume anonymous). + overwrite: Overwrite destination or not. + """ + mdio.copy_mdio( + source=input_mdio_path, + dest_path_or_buffer=output_mdio_path, + excludes=excludes, + includes=includes, + storage_options=storage_options, + overwrite=overwrite, + ) + + +@cli.command(name="info") +@click.option( + "-i", + "--input-mdio-file", + required=True, + help="Input path of the mdio file", + type=click.STRING, +) +@click.option( + "-format", + "--output-format", + required=False, + default="plain", + help="Output format, plain is human readable. JSON will output in json format for easier passing. ", + type=click.Choice(["plain", "json"]), + show_default=True, + show_choices=True, +) +def info( + input_mdio_file, + output_format, +): + """Provide information on MDIO dataset. + + By default this returns human readable information about the grid and stats for the dataset. If output-format is set to json then a json is returned to facilitate parsing + + + """ + reader = mdio.MDIOReader(input_mdio_file, return_metadata=True) + mdio_dict = {} + mdio_dict["grid"] = {} + for axis in reader.grid.dim_names: + dim = reader.grid.select_dim(axis) + min = dim.coords[0] + max = dim.coords[-1] + size = dim.coords.shape[0] + axis_dict = {"name": axis, "min": min, "max": max, "size": size} + mdio_dict["grid"][axis] = axis_dict + + if output_format == "plain": + click.echo("{:<10} {:<10} {:<10} {:<10}".format("NAME", "MIN", "MAX", "SIZE")) + click.echo("=" * 40) + + for _, axis_dict in mdio_dict["grid"].items(): + click.echo( + "{:<10} {:<10} {:<10} {:<10}".format( + axis_dict["name"], + axis_dict["min"], + axis_dict["max"], + axis_dict["size"], + ) + ) + + click.echo("\n\n{:<10} {:<10}".format("STAT", "VALUE")) + click.echo("=" * 20) + for name, stat in reader.stats.items(): + click.echo("{:<10} {:<10}".format(name, stat)) + if output_format == "json": + mdio_dict["stats"] = reader.stats + click.echo(mdio_dict) From b1c2315348ac90d23df7cf393d225a806f166aba Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Wed, 21 Jun 2023 08:23:56 -0500 Subject: [PATCH 02/10] Improve formating --- src/mdio/commands/utility.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/mdio/commands/utility.py b/src/mdio/commands/utility.py index 8e19e9c8..0929a578 100644 --- a/src/mdio/commands/utility.py +++ b/src/mdio/commands/utility.py @@ -40,14 +40,19 @@ def cli(): "-exc", "--excludes", required=False, - help="Data to exclude during copy. i.e. chunked_012. The raw data won’t be copied, but it will create an empty array to be filled. If left blank, it will copy everything.", + help="""Data to exclude during copy. i.e. chunked_012. The raw data won’t be + copied, but it will create an empty array to be filled. If left blank, it will + copy everything.""", type=click.STRING, ) @click.option( "-inc", "--includes", required=False, - help="Data to include during copy. i.e. trace_headers. If this is not specified, and certain data is excluded, it will not copy headers. If you want to preserve headers, specify trace_headers. If left blank, it will copy everything except specified in excludes parameter.", + help="""Data to include during copy. i.e. trace_headers. If this is not + specified, and certain data is excluded, it will not copy headers. If you want + to preserve headers, specify trace_headers. If left blank, it will copy + everything except specified in excludes parameter.""", type=click.STRING, ) @click.option( @@ -118,7 +123,8 @@ def copy( "--output-format", required=False, default="plain", - help="Output format, plain is human readable. JSON will output in json format for easier passing. ", + help="""Output format, plain is human readable. JSON will output in json + format for easier passing. """, type=click.Choice(["plain", "json"]), show_default=True, show_choices=True, @@ -129,9 +135,9 @@ def info( ): """Provide information on MDIO dataset. - By default this returns human readable information about the grid and stats for the dataset. If output-format is set to json then a json is returned to facilitate parsing - - + By default this returns human readable information about the grid and stats for + the dataset. If output-format is set to json then a json is returned to + facilitate parsing. """ reader = mdio.MDIOReader(input_mdio_file, return_metadata=True) mdio_dict = {} From 65b2bca9b53100225c995d50356546083cfd49c8 Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Wed, 21 Jun 2023 11:23:35 -0500 Subject: [PATCH 03/10] Updates to pass pre-commit. --- src/mdio/commands/utility.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/mdio/commands/utility.py b/src/mdio/commands/utility.py index 0929a578..60a76ccc 100644 --- a/src/mdio/commands/utility.py +++ b/src/mdio/commands/utility.py @@ -6,19 +6,19 @@ import click_params import mdio - import json except SystemError: pass DEFAULT_HELP = """ -MDIO CLI utilities. +MDIO CLI utilities. """ @click.group(help=DEFAULT_HELP) def cli(): - click.echo(f"MDIO CLI utilities") + """Setup click group.""" + click.echo("MDIO CLI utilities") @cli.command(name="copy") @@ -49,7 +49,7 @@ def cli(): "-inc", "--includes", required=False, - help="""Data to include during copy. i.e. trace_headers. If this is not + help="""Data to include during copy. i.e. trace_headers. If this is not specified, and certain data is excluded, it will not copy headers. If you want to preserve headers, specify trace_headers. If left blank, it will copy everything except specified in excludes parameter.""", @@ -80,6 +80,7 @@ def copy( overwrite: bool = False, ): """Copy MDIO to MDIO. + Can also copy with empty data to be filled later. See `excludes` and `includes` parameters. @@ -87,8 +88,8 @@ def copy( in Zarr's documentation in `zarr.convenience.copy_store`. Args: - source: MDIO reader or accessor instance. Data will be copied from here - dest_path_or_buffer: Destination path. Could be any FSSpec mapping. + input_mdio_path: MDIO reader or accessor instance. Data will be copied from here + output_mdio_path: Destination path. Could be any FSSpec mapping. excludes: Data to exclude during copy. i.e. `chunked_012`. The raw data won't be copied, but it will create an empty array to be filled. If left blank, it will copy everything. @@ -123,7 +124,7 @@ def copy( "--output-format", required=False, default="plain", - help="""Output format, plain is human readable. JSON will output in json + help="""Output format, plain is human readable. JSON will output in json format for easier passing. """, type=click.Choice(["plain", "json"]), show_default=True, @@ -167,7 +168,7 @@ def info( click.echo("\n\n{:<10} {:<10}".format("STAT", "VALUE")) click.echo("=" * 20) for name, stat in reader.stats.items(): - click.echo("{:<10} {:<10}".format(name, stat)) + click.echo(f"{name:<10} {stat:<10}") if output_format == "json": mdio_dict["stats"] = reader.stats click.echo(mdio_dict) From 4cffc85bfa3104ee4e4981f1d30dea4f345e4cfc Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Thu, 29 Jun 2023 09:03:43 -0500 Subject: [PATCH 04/10] Add access pattern to info. --- src/mdio/commands/utility.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/mdio/commands/utility.py b/src/mdio/commands/utility.py index 60a76ccc..9b0d3d78 100644 --- a/src/mdio/commands/utility.py +++ b/src/mdio/commands/utility.py @@ -119,6 +119,15 @@ def copy( help="Input path of the mdio file", type=click.STRING, ) +@click.option( + "-access", + "--access-pattern", + required=False, + default="012", + help="Access pattern of the file", + type=click.STRING, + show_default=True, +) @click.option( "-format", "--output-format", @@ -133,6 +142,7 @@ def copy( def info( input_mdio_file, output_format, + access_pattern, ): """Provide information on MDIO dataset. @@ -140,7 +150,9 @@ def info( the dataset. If output-format is set to json then a json is returned to facilitate parsing. """ - reader = mdio.MDIOReader(input_mdio_file, return_metadata=True) + reader = mdio.MDIOReader( + input_mdio_file, access_pattern=access_pattern, return_metadata=True + ) mdio_dict = {} mdio_dict["grid"] = {} for axis in reader.grid.dim_names: From 3a7e4dc786c5745521068cab487070868ec6ae06 Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Thu, 6 Jul 2023 13:48:13 -0500 Subject: [PATCH 05/10] Add tests. --- .gitignore | 4 +++- tests/conftest.py | 7 +++++++ tests/test_main.py | 23 +++++++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f67d2adb..7d068c87 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,9 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST - +pip-* +*pytest* +tmp* # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. diff --git a/tests/conftest.py b/tests/conftest.py index 9015bd95..19e9e851 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,13 @@ def zarr_tmp(tmp_path_factory): return tmp_file +@pytest.fixture(scope="session") +def zarr_tmp2(tmp_path_factory): + """Make a temp file for the output MDIO.""" + tmp_file = tmp_path_factory.mktemp(r"mdio2") + return tmp_file + + @pytest.fixture(scope="session") def segy_export_ibm_tmp(tmp_path_factory): """Make a temp file for the round-trip IBM SEG-Y.""" diff --git a/tests/test_main.py b/tests/test_main.py index ac91dfe5..c2e88b3e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -15,6 +15,7 @@ def runner() -> CliRunner: return CliRunner() +@pytest.mark.dependency() def test_main_succeeds(runner: CliRunner, segy_input: str, zarr_tmp: Path) -> None: """It exits with a status code of zero.""" cli_args = ["segy", "import"] @@ -27,6 +28,28 @@ def test_main_succeeds(runner: CliRunner, segy_input: str, zarr_tmp: Path) -> No assert result.exit_code == 0 +@pytest.mark.dependency(depends=["test_main_succeeds"]) +def test_main_info_succeeds(runner: CliRunner, zarr_tmp: Path) -> None: + """It exits with a status code of zero.""" + cli_args = ["utility", "info"] + cli_args.extend(["-i", str(zarr_tmp)]) + + result = runner.invoke(__main__.main, args=cli_args) + assert result.exit_code == 0 + + +@pytest.mark.dependency(depends=["test_main_succeeds"]) +def test_main_copy_succeeds(runner: CliRunner, zarr_tmp: Path, zarr_tmp2: Path) -> None: + """It exits with a status code of zero.""" + cli_args = ["utility", "copy"] + cli_args.extend(["-i", str(zarr_tmp)]) + cli_args.extend(["-o", str(zarr_tmp2)]) + + result = runner.invoke(__main__.main, args=cli_args) + print(f"copy returns {result}") + # assert result.exit_code == 0 + + def test_cli_version(runner: CliRunner) -> None: """Check if version prints without error.""" cli_args = ["--version"] From 784669bfe5ffcc4b296865076ea53ab9b822db01 Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Wed, 19 Jul 2023 15:55:23 -0500 Subject: [PATCH 06/10] Move utility commands to root level. --- src/mdio/__main__.py | 14 ++++++++++++-- src/mdio/commands/utility.py | 8 ++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/mdio/__main__.py b/src/mdio/__main__.py index 10e3ae3e..4223f383 100644 --- a/src/mdio/__main__.py +++ b/src/mdio/__main__.py @@ -11,6 +11,7 @@ import mdio +from mdio.commands.utility import copy, info plugin_folder = os.path.join(os.path.dirname(__file__), "commands") @@ -40,11 +41,18 @@ class MyCLI(click.MultiCommand): http://lybniz2.sourceforge.net/safeeval.html """ + _command_mapping = { + "copy": copy, + "info": info, + } + + protected_files = ["__init__.py", "utility.py"] + def list_commands(self, ctx: click.Context) -> list[str]: """List commands available under `commands` module.""" - rv = [] + rv = list(self._command_mapping.keys()) for filename in os.listdir(plugin_folder): - if filename.endswith(".py") and filename != "__init__.py": + if filename.endswith(".py") and filename not in self.protected_files: rv.append(filename[:-3]) rv.sort() @@ -61,6 +69,8 @@ def get_command(self, ctx: click.Context, name: str) -> dict[Callable]: } local_ns = {} + if name in self._command_mapping.keys(): + return self._command_mapping[name] fn = os.path.join(plugin_folder, name + ".py") with open(fn) as f: code = compile(f.read(), fn, "exec") diff --git a/src/mdio/commands/utility.py b/src/mdio/commands/utility.py index 9b0d3d78..595a2d1a 100644 --- a/src/mdio/commands/utility.py +++ b/src/mdio/commands/utility.py @@ -1,11 +1,11 @@ """Dataset CLI Plugin.""" - try: import click import click_params import mdio + from typing import Optional except SystemError: pass @@ -21,7 +21,7 @@ def cli(): click.echo("MDIO CLI utilities") -@cli.command(name="copy") +@click.command(name="copy") @click.option( "-i", "--input-mdio-path", @@ -76,7 +76,7 @@ def copy( output_mdio_path: str, includes: str = "", excludes: str = "", - storage_options: dict | None = None, + storage_options: Optional[dict] = None, overwrite: bool = False, ): """Copy MDIO to MDIO. @@ -111,7 +111,7 @@ def copy( ) -@cli.command(name="info") +@click.command(name="info") @click.option( "-i", "--input-mdio-file", From 4052356dd47569481504c514250428c161178f18 Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Wed, 19 Jul 2023 16:42:37 -0500 Subject: [PATCH 07/10] Update linting and tests. --- src/mdio/__main__.py | 3 ++- src/mdio/commands/utility.py | 3 ++- tests/test_main.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/mdio/__main__.py b/src/mdio/__main__.py index 4223f383..5234ff73 100644 --- a/src/mdio/__main__.py +++ b/src/mdio/__main__.py @@ -10,8 +10,9 @@ import click_params import mdio +from mdio.commands.utility import copy +from mdio.commands.utility import info -from mdio.commands.utility import copy, info plugin_folder = os.path.join(os.path.dirname(__file__), "commands") diff --git a/src/mdio/commands/utility.py b/src/mdio/commands/utility.py index 595a2d1a..8f9f6721 100644 --- a/src/mdio/commands/utility.py +++ b/src/mdio/commands/utility.py @@ -1,11 +1,12 @@ """Dataset CLI Plugin.""" try: + from typing import Optional + import click import click_params import mdio - from typing import Optional except SystemError: pass diff --git a/tests/test_main.py b/tests/test_main.py index c2e88b3e..6c7daeb5 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -31,7 +31,7 @@ def test_main_succeeds(runner: CliRunner, segy_input: str, zarr_tmp: Path) -> No @pytest.mark.dependency(depends=["test_main_succeeds"]) def test_main_info_succeeds(runner: CliRunner, zarr_tmp: Path) -> None: """It exits with a status code of zero.""" - cli_args = ["utility", "info"] + cli_args = ["info"] cli_args.extend(["-i", str(zarr_tmp)]) result = runner.invoke(__main__.main, args=cli_args) @@ -41,7 +41,7 @@ def test_main_info_succeeds(runner: CliRunner, zarr_tmp: Path) -> None: @pytest.mark.dependency(depends=["test_main_succeeds"]) def test_main_copy_succeeds(runner: CliRunner, zarr_tmp: Path, zarr_tmp2: Path) -> None: """It exits with a status code of zero.""" - cli_args = ["utility", "copy"] + cli_args = ["copy"] cli_args.extend(["-i", str(zarr_tmp)]) cli_args.extend(["-o", str(zarr_tmp2)]) From 60a6ec5a8791af01e788c64f48d56cf28709aa1f Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Tue, 7 Nov 2023 15:30:43 -0600 Subject: [PATCH 08/10] Resolve PR comments. ments. # Please enter the commit message for your changes. Lines starting # with '#' will be ignored, and an empty message aborts the commit. # # interactive rebase in progress; onto 2fa8069 # Last commands done (3 commands done): # pick 7db1c77 Updates to pass pre-commit. # pick 9197e4e Bug fix and log statements. # Next commands to do (2 remaining commands): # pick f382208 Updatest to resolve conflicts with changes related to floating point headers for indexing. # pick 6961e1d Fix pre-commit linting. # You are currently rebasing branch 'main' on '2fa8069'. # # Changes to be committed: # modified: src/mdio/segy/utilities.py # modified: tests/integration/test_segy_import_export.py # # Untracked files: # .devcontainer_bck/Dockerfile # .devcontainer_bck/devcontainer.json # .devcontainer_bck/post-install.sh # .gitignore_bck # TEST.md # debugging/0__6372_PREPNODMLT.LIBRARY_2.mdio.jpg # debugging/6372_export.py # debugging/6372_ram_usage.csv # debugging/6372_ram_usage.csv_bck # debugging/Debug.md # debugging/None # debugging/docker_env/bin/Activate.ps1 # debugging/docker_env/bin/activate # debugging/docker_env/bin/activate.csh # debugging/docker_env/bin/activate.fish # debugging/docker_env/bin/dask # debugging/docker_env/bin/dask-scheduler # debugging/docker_env/bin/dask-ssh # debugging/docker_env/bin/dask-worker # debugging/docker_env/bin/doesitcache # debugging/docker_env/bin/dul-receive-pack # debugging/docker_env/bin/dul-upload-pack # debugging/docker_env/bin/dulwich # debugging/docker_env/bin/easy_install # debugging/docker_env/bin/easy_install-3.8 # debugging/docker_env/bin/f2py # debugging/docker_env/bin/f2py3 # debugging/docker_env/bin/f2py3.8 # debugging/docker_env/bin/fonttools # debugging/docker_env/bin/jsonschema # debugging/docker_env/bin/keyring # debugging/docker_env/bin/pip # debugging/docker_env/bin/pip3 # debugging/docker_env/bin/pip3.8 # debugging/docker_env/bin/pkginfo # debugging/docker_env/bin/poetry # debugging/docker_env/bin/py-spy # debugging/docker_env/bin/pyftmerge # debugging/docker_env/bin/pyftsubset # debugging/docker_env/bin/pyproject-build # debugging/docker_env/bin/python # debugging/docker_env/bin/python3 # debugging/docker_env/bin/ttx # debugging/docker_env/bin/virtualenv # debugging/docker_env/lib64 # debugging/docker_env/share/man/man1/ttx.1 # debugging/docker_env/share/python-wheels/CacheControl-0.12.6-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/appdirs-1.4.3-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/certifi-2019.11.28-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/chardet-3.0.4-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/colorama-0.4.3-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/contextlib2-0.6.0-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/distlib-0.3.0-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/distro-1.4.0-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/html5lib-1.0.1-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/idna-2.8-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/ipaddr-2.2.0-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/lockfile-0.12.2-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/msgpack-0.6.2-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/packaging-20.3-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/pep517-0.8.2-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/pip-20.0.2-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/pkg_resources-0.0.0-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/progress-1.5-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/pyparsing-2.4.6-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/requests-2.22.0-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/retrying-1.3.3-py2.py3-none-any.whl # debugging/docker_env/share/python-wheels/setuptools-44.0.0-py2.py3-none-any.whl # deb --- .gitignore | 3 +-- src/mdio/commands/utility.py | 33 ++++----------------------------- tests/test_main.py | 3 +-- 3 files changed, 6 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 7d068c87..41c8888e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,6 @@ share/python-wheels/ *.egg MANIFEST pip-* -*pytest* tmp* # PyInstaller # Usually these files are written by a python script from a template @@ -152,4 +151,4 @@ cython_debug/ mdio1/* */mdio1/* pytest-of-* -tmp/ +tmp diff --git a/src/mdio/commands/utility.py b/src/mdio/commands/utility.py index 8f9f6721..f4a02dae 100644 --- a/src/mdio/commands/utility.py +++ b/src/mdio/commands/utility.py @@ -11,17 +11,6 @@ pass -DEFAULT_HELP = """ -MDIO CLI utilities. -""" - - -@click.group(help=DEFAULT_HELP) -def cli(): - """Setup click group.""" - click.echo("MDIO CLI utilities") - - @click.command(name="copy") @click.option( "-i", @@ -51,8 +40,8 @@ def cli(): "--includes", required=False, help="""Data to include during copy. i.e. trace_headers. If this is not - specified, and certain data is excluded, it will not copy headers. If you want - to preserve headers, specify trace_headers. If left blank, it will copy + specified, and certain data is excluded, it will not copy headers. To + preserve headers, specify trace_headers. If left blank, it will copy everything except specified in excludes parameter.""", type=click.STRING, ) @@ -80,27 +69,13 @@ def copy( storage_options: Optional[dict] = None, overwrite: bool = False, ): - """Copy MDIO to MDIO. + """Copy a MDIO dataset to anpther MDIO dataset. Can also copy with empty data to be filled later. See `excludes` and `includes` parameters. More documentation about `excludes` and `includes` can be found in Zarr's documentation in `zarr.convenience.copy_store`. - - Args: - input_mdio_path: MDIO reader or accessor instance. Data will be copied from here - output_mdio_path: Destination path. Could be any FSSpec mapping. - excludes: Data to exclude during copy. i.e. `chunked_012`. The raw data - won't be copied, but it will create an empty array to be filled. - If left blank, it will copy everything. - includes: Data to include during copy. i.e. `trace_headers`. If this is - not specified, and certain data is excluded, it will not copy headers. - If you want to preserve headers, specify `trace_headers`. If left blank, - it will copy everything except specified in `excludes` parameter. - storage_options: Storage options for the cloud storage backend. - Default is None (will assume anonymous). - overwrite: Overwrite destination or not. """ mdio.copy_mdio( source=input_mdio_path, @@ -145,7 +120,7 @@ def info( output_format, access_pattern, ): - """Provide information on MDIO dataset. + """Provide information on a MDIO dataset. By default this returns human readable information about the grid and stats for the dataset. If output-format is set to json then a json is returned to diff --git a/tests/test_main.py b/tests/test_main.py index 6c7daeb5..87d2787d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -46,8 +46,7 @@ def test_main_copy_succeeds(runner: CliRunner, zarr_tmp: Path, zarr_tmp2: Path) cli_args.extend(["-o", str(zarr_tmp2)]) result = runner.invoke(__main__.main, args=cli_args) - print(f"copy returns {result}") - # assert result.exit_code == 0 + assert result.exit_code == 0 def test_cli_version(runner: CliRunner) -> None: From 267df52afa0dde29972941d10a5f2fee48df5e12 Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Tue, 7 Nov 2023 16:51:35 -0600 Subject: [PATCH 09/10] Fix mdio copy. --- noxfile.py | 1 + src/mdio/commands/utility.py | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index ac271f2f..291167a5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -132,6 +132,7 @@ def precommit(session: Session) -> None: "pre-commit", "pre-commit-hooks", "pyupgrade", + "pytest-dependency", ) session.run("pre-commit", *args) if args and args[0] == "install": diff --git a/src/mdio/commands/utility.py b/src/mdio/commands/utility.py index f4a02dae..e3bb7628 100644 --- a/src/mdio/commands/utility.py +++ b/src/mdio/commands/utility.py @@ -26,10 +26,20 @@ help="Output path or URL to write the mdio dataset.", type=click.STRING, ) +@click.option( + "-access", + "--access-pattern", + required=False, + default="012", + help="Access pattern of the file", + type=click.STRING, + show_default=True, +) @click.option( "-exc", "--excludes", required=False, + default="", help="""Data to exclude during copy. i.e. chunked_012. The raw data won’t be copied, but it will create an empty array to be filled. If left blank, it will copy everything.""", @@ -39,6 +49,7 @@ "-inc", "--includes", required=False, + default="", help="""Data to include during copy. i.e. trace_headers. If this is not specified, and certain data is excluded, it will not copy headers. To preserve headers, specify trace_headers. If left blank, it will copy @@ -62,8 +73,9 @@ show_default=True, ) def copy( - input_mdio_path, + input_mdio_path: str, output_mdio_path: str, + access_pattern: str = "012", includes: str = "", excludes: str = "", storage_options: Optional[dict] = None, @@ -77,8 +89,11 @@ def copy( More documentation about `excludes` and `includes` can be found in Zarr's documentation in `zarr.convenience.copy_store`. """ + reader = mdio.MDIOReader( + input_mdio_path, access_pattern=access_pattern, return_metadata=True + ) mdio.copy_mdio( - source=input_mdio_path, + source=reader, dest_path_or_buffer=output_mdio_path, excludes=excludes, includes=includes, From fe08fdf59ab43a069c8b2445a9afd78fb84c3026 Mon Sep 17 00:00:00 2001 From: Mark Roberts Date: Tue, 7 Nov 2023 17:04:08 -0600 Subject: [PATCH 10/10] Remove duplicate tmp in .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 41c8888e..bfdc38f2 100644 --- a/.gitignore +++ b/.gitignore @@ -152,3 +152,4 @@ mdio1/* */mdio1/* pytest-of-* tmp +debugging/*