Skip to content

Commit

Permalink
Add Gen3 Workflow cli and tool. (#244)
Browse files Browse the repository at this point in the history
* Add cli and tool logic

* Add unit tests

* Add Documentation
  • Loading branch information
nss10 authored Dec 11, 2024
1 parent 3a6dce7 commit 1091867
Show file tree
Hide file tree
Showing 12 changed files with 97 additions and 5 deletions.
4 changes: 2 additions & 2 deletions .secrets.baseline
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@
"filename": "docs/reference/sdkClasses.md",
"hashed_secret": "749dcc9e92f723c96ed4a3a908ded1bc559c66a9",
"is_verified": false,
"line_number": 58
"line_number": 63
}
],
"docs/tutorial/quickStart.md": [
Expand Down Expand Up @@ -656,5 +656,5 @@
}
]
},
"generated_at": "2024-06-10T19:49:24Z"
"generated_at": "2024-12-05T20:51:45Z"
}
Binary file modified docs/_build/doctrees/environment.pickle
Binary file not shown.
Binary file modified docs/_build/doctrees/tools/indexing.doctree
Binary file not shown.
Binary file modified docs/_build/doctrees/tools/metadata.doctree
Binary file not shown.
2 changes: 1 addition & 1 deletion docs/_build/html/searchindex.js

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion docs/_build/html/tools/indexing.html
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ <h1>Indexing Tools<a class="headerlink" href="#indexing-tools" title="Link to th

<dl class="py function">
<dt class="sig sig-object py" id="gen3.tools.indexing.verify_manifest.async_verify_object_manifest">
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1729629627.0084667.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Link to this definition"></a></dt>
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1733932163.2023835.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Link to this definition"></a></dt>
<dd><p>Verify all file object records into a manifest csv</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
Expand Down
2 changes: 1 addition & 1 deletion docs/_build/html/tools/metadata.html
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ <h1>Metadata Tools<a class="headerlink" href="#metadata-tools" title="Link to th

<dl class="py function">
<dt class="sig sig-object py" id="gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest">
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1729629627.256328.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Link to this definition"></a></dt>
<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1733932163.4401503.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Link to this definition"></a></dt>
<dd><p>Ingest all metadata records into a manifest csv</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
Expand Down
5 changes: 5 additions & 0 deletions docs/reference/sdkClasses.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ TOC
- [Gen3Jobs](#gen3jobs)
- Gen3File
- [Gen3Query](#gen3query)
- [Gen3Wrap](#gen3wrap)
- Gen3Metadata
- Gen3WsStorage

Expand Down Expand Up @@ -35,6 +36,10 @@ This is the client for interacting with the Indexd service for GUID brokering an

This is the client for interacting with the Gen3 submission service including GraphQL queries.

### Gen3Wrap

A class that leverages `Gen3Auth` to retrieve the access token from the user's `~/.gen3/credentials.json` file, sets it as the `GEN3_TOKEN` environment variable, and relays all commands and options passed to `gen3 run`

### Gen3Query

This is the client for interacting with the Gen3 ElasticSearch query service.
Expand Down
2 changes: 2 additions & 0 deletions gen3/cli/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import gen3.cli.file as file
import gen3.cli.drs_pull as drs_pull
import gen3.cli.users as users
import gen3.cli.wrap as wrap
import gen3
from gen3 import logging as sdklogging
from gen3.cli import nih
Expand Down Expand Up @@ -141,4 +142,5 @@ def main(
main.add_command(file.file)
main.add_command(nih.nih)
main.add_command(users.users)
main.add_command(wrap.run)
main()
16 changes: 16 additions & 0 deletions gen3/cli/wrap.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import click


from gen3.tools.wrap import Gen3Wrap


@click.command(
context_settings={"ignore_unknown_options": True, "allow_extra_args": True},
help="A wrapper command that forwards COMMAND_ARGS as-is after setting the environment variable GEN3_TOKEN",
)
@click.argument("command_args", nargs=-1, type=click.UNPROCESSED)
@click.pass_context
def run(ctx, command_args):
auth = ctx.obj["auth_factory"].get()
gen3Wrap_object = Gen3Wrap(auth, command_args)
gen3Wrap_object.run_command()
36 changes: 36 additions & 0 deletions gen3/tools/wrap.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import os
import subprocess

from cdislogging import get_logger
from gen3.auth import Gen3Auth, Gen3AuthError

logger = get_logger("__name__")


class Gen3Wrap:
def __init__(self, auth: Gen3Auth, command_args: tuple):
"""
auth : Gen3Auth instance
command_args: A tuple consisting of all the commands sent to the `gen3 run` tool
"""
self.auth = auth
self.command_args = command_args

def run_command(self):
"""
Take the command args and run a subprocess with appropriate access token in the env var
"""
cmd = list(self.command_args)
try:
os.environ["GEN3_TOKEN"] = self.auth.get_access_token()
except Gen3AuthError as e:
logger.error(f"ERROR getting Gen3 Access Token:", e)
raise
logger.info(
f"Running the command {self.command_args} with gen3 access token in environment variable"
)
try:
subprocess.run(cmd, stderr=subprocess.STDOUT)
except Exception as e:
logger.error(f"ERROR while running '{cmd}':", e)
raise
33 changes: 33 additions & 0 deletions tests/test_wrap.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import pytest
from unittest.mock import MagicMock, patch

from gen3.tools.wrap import Gen3Wrap, Gen3Auth, Gen3AuthError


@patch.object(Gen3Auth, "get_access_token", MagicMock(return_value="1.2.3"))
def test_gen3_wrap_valid_auth(mock_gen3_auth):
"""
Patch subprocess.run and verify that the appropriate arguments are passed to the method when authentication process is valid.
"""

test_command_args = ("echo", "Test1", "Test2")
with patch("gen3.tools.wrap.subprocess.run") as mock_subprocess_run:
wrapper_obj = Gen3Wrap(mock_gen3_auth, test_command_args)
wrapper_obj.run_command()
mock_subprocess_run.assert_called_once_with(list(test_command_args), stderr=-2)


@patch.object(Gen3Auth, "get_access_token", MagicMock(side_effect=Gen3AuthError()))
def test_gen3_wrap_inavalid_auth(mock_gen3_auth):
"""
Break the authentication process to verify the following:
1. Ensure a Gen3AuthError is raised.
2. Confirm the subprocess is not executed when authentication fails.
"""

test_command_args = ("echo", "Test1", "Test2")
with pytest.raises(Gen3AuthError):
with patch("gen3.tools.wrap.subprocess.run") as mock_subprocess_run:
wrapper_obj = Gen3Wrap(mock_gen3_auth, test_command_args)
wrapper_obj.run_command()
mock_subprocess_run.assert_not_called()

0 comments on commit 1091867

Please sign in to comment.