chore(deps): update deps, docs, and use common utils function (#137)

* chore(deps): update deps, docs, and use common utils function

* Apply automatic documentation changes

* chore(version): bump

* Apply automatic documentation changes

Co-authored-by: Alexander VT <[email protected]>

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "(docs\\/_build|poetry.lock)|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2022-06-07T20:32:38Z",
+  "generated_at": "2022-06-10T15:20:47Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -62,19 +62,19 @@
       {
         "hashed_secret": "7c35c215b326b9463b669b657c1ff9873ff53d9a",
         "is_verified": false,
-        "line_number": 202,
+        "line_number": 203,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "0d515eaf06062d52e8c80abb4d3b713a65396d30",
         "is_verified": false,
-        "line_number": 207,
+        "line_number": 208,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "b4cff7c2af45cdfe66195ec574a7b8832f8621ea",
         "is_verified": false,
-        "line_number": 214,
+        "line_number": 215,
         "type": "Hex High Entropy String"
       }
     ],
@@ -350,49 +350,49 @@
       {
         "hashed_secret": "96c9184fb19c9c1618ccf44d141f8029a739891c",
         "is_verified": false,
-        "line_number": 426,
+        "line_number": 424,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "e1da93616713812cb50e0ac845b1e9e305d949f1",
         "is_verified": false,
-        "line_number": 562,
+        "line_number": 559,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "47f42f4c34fddab383b817e689dc0fb75af81266",
         "is_verified": false,
-        "line_number": 586,
+        "line_number": 583,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "300d95dd5d30ab6928ffda6c08c6a129a23e5b39",
         "is_verified": false,
-        "line_number": 610,
+        "line_number": 607,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "f9e664db75c7f23a299b0b055c10e08d47073e93",
         "is_verified": false,
-        "line_number": 678,
+        "line_number": 675,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "7c35c215b326b9463b669b657c1ff9873ff53d9a",
         "is_verified": false,
-        "line_number": 819,
+        "line_number": 816,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "0d515eaf06062d52e8c80abb4d3b713a65396d30",
         "is_verified": false,
-        "line_number": 823,
+        "line_number": 820,
         "type": "Hex High Entropy String"
       },
       {
         "hashed_secret": "b4cff7c2af45cdfe66195ec574a7b8832f8621ea",
         "is_verified": false,
-        "line_number": 828,
+        "line_number": 825,
         "type": "Hex High Entropy String"
       }
     ],

docs/_build/html/tools/indexing.html

@@ -340,7 +340,7 @@ <h1>Indexing Tools<a class="headerlink" href="#indexing-tools" title="Permalink
 
 <dl class="py function">
 <dt class="sig sig-object py" id="gen3.tools.indexing.verify_manifest.async_verify_object_manifest">
-<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1654807944.3028057.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
+<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></span><span class="sig-name descname"><span class="pre">async_verify_object_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='verify-manifest-errors-1654874771.190345.log'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
 <dd><p>Verify all file object records into a manifest csv</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>

docs/_build/html/tools/metadata.html

@@ -104,7 +104,7 @@ <h1>Metadata Tools<a class="headerlink" href="#metadata-tools" title="Permalink
 
 <dl class="py function">
 <dt class="sig sig-object py" id="gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest">
-<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1654807944.560571.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
+<em class="property"><span class="k"><span class="pre">async</span></span><span class="w"> </span></em><span class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></span><span class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">commons_url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_source</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">auth=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_concurrent_requests=24</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">manifest_file_delimiter=None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">output_filename='ingest-metadata-manifest-errors-1654874771.4957037.log'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">get_guid_from_file=True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">metadata_type=None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
 <dd><p>Ingest all metadata records into a manifest csv</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>

docs/howto/diirmIndexing.md

@@ -18,15 +18,16 @@ import asyncio
 
 from gen3.tools import indexing
 from gen3.tools.indexing.verify_manifest import manifest_row_parsers
+from gen3.utils import get_or_create_event_loop_for_thread
 
 logging.basicConfig(filename="output.log", level=logging.DEBUG)
 logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
 
 COMMONS = "https://{{insert-commons-here}}/"
 
 def main():
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
+
 
     loop.run_until_complete(
         indexing.async_download_object_manifest(
@@ -83,8 +84,8 @@ def main():
     # override default parsers
     manifest_row_parsers["file_size"] = _get_file_size
 
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
+
 
     loop.run_until_complete(
         indexing.async_verify_object_manifest(

docs/howto/metadataTools.md

@@ -30,6 +30,7 @@ import asyncio
 from gen3.auth import Gen3Auth
 from gen3.tools import metadata
 from gen3.tools.metadata.ingest_manifest import manifest_row_parsers
+from gen3.utils import get_or_create_event_loop_for_thread
 
 logging.basicConfig(filename="output.log", level=logging.DEBUG)
 logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
@@ -39,8 +40,8 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
 MANIFEST = "dbgap_extract_guid.tsv"
 
 def main():
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
+
 
     auth = Gen3Auth(refresh_file="credentials.json")
 
@@ -256,8 +257,8 @@ MANIFEST = "dbgap_extract.tsv"
 
 
 def main():
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
+
 
     auth = Gen3Auth(refresh_file="credentials.json")
 
@@ -565,8 +566,8 @@ def get_dbgap_merged_metadata_manifest():
         "partial_match_or_exact_match": "partial_match",
     }
 
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
+
 
     job_output = loop.run_until_complete(
         jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=job_input)
@@ -591,8 +592,8 @@ def metadata_ingest():
         "metadata_source": "dbgap",
     }
 
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
+
 
     job_output = loop.run_until_complete(
         jobs.async_run_job_and_wait(job_name=INGEST_METADATA_JOB, job_input=job_input)
@@ -633,8 +634,8 @@ COMMONS = "https://{{insert-commons-here}}/"
 
 
 def main():
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
+
 
     loop.run_until_complete(
         metadata.async_verify_metadata_manifest(

docs/reference/sdkClasses.md

@@ -17,7 +17,7 @@ This contains an auth wrapper for supporting JWT based authentication with `requ
 By default - the `Gen3Auth` constructor looks for an api key
 in `~/.gen3/credentials.json`.  You may override that path
 via the `GEN3_API_KEY` environment varialbe, or by passing a
-`refresh_file` parameter. 
+`refresh_file` parameter.
 
 When working in a Gen3 Workspace, all parameters are optional and the `Gen3Auth` instance should be initialized as follows:
 
@@ -52,6 +52,7 @@ import asyncio
 from gen3.index import Gen3Index
 from gen3.auth import Gen3Auth
 from gen3.jobs import Gen3Jobs, DBGAP_METADATA_JOB, INGEST_METADATA_JOB
+from gen3.utils import get_or_create_event_loop_for_thread
 
 # An API Key downloaded from the above commons' "Profile" page
 API_KEY_FILEPATH = "credentials.json"
@@ -69,8 +70,7 @@ def metadata_ingest():
         "metadata_source": "dbgaptest",
     }
 
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
 
     job_output = loop.run_until_complete(
         jobs.async_run_job_and_wait(job_name=INGEST_METADATA_JOB, job_input=job_input)
@@ -93,8 +93,7 @@ def main():
         "partial_match_or_exact_match": "partial_match",
     }
 
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
 
     job_output = loop.run_until_complete(
         jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=job_input)
@@ -138,8 +137,7 @@ def example_async_run_job():
     auth = Gen3Auth(refresh_file=API_KEY_FILEPATH)
     jobs = Gen3Jobs(auth_provider=auth)
 
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
+    loop = get_or_create_event_loop_for_thread()
 
     job_output = loop.run_until_complete(
         jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=JOB_INPUT)