(PXP:8581): Add apiEndpoint and other arguments to configure to avoid breaking changes

docs/_build/html/tools/indexing.html

@@ -365,7 +365,7 @@ <h1>Indexing Tools<a class="headerlink" href="#indexing-tools" title="Permalink
 
 <dl class="py function">
 <dt id="gen3.tools.indexing.verify_manifest.async_verify_object_manifest">
-<em class="property"><span class="pre">async</span> </em><code class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></code><code class="sig-name descname"><span class="pre">async_verify_object_manifest</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="pre">commons_url</span></em>, <em class="sig-param"><span class="pre">manifest_file</span></em>, <em class="sig-param"><span class="pre">max_concurrent_requests=24</span></em>, <em class="sig-param"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></em>, <em class="sig-param"><span class="pre">manifest_file_delimiter=None</span></em>, <em class="sig-param"><span class="pre">output_filename='verify-manifest-errors-1630075391.7987797.log'</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
+<em class="property"><span class="pre">async</span> </em><code class="sig-prename descclassname"><span class="pre">gen3.tools.indexing.verify_manifest.</span></code><code class="sig-name descname"><span class="pre">async_verify_object_manifest</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="pre">commons_url</span></em>, <em class="sig-param"><span class="pre">manifest_file</span></em>, <em class="sig-param"><span class="pre">max_concurrent_requests=24</span></em>, <em class="sig-param"><span class="pre">manifest_row_parsers={'acl':</span> <span class="pre">&lt;function</span> <span class="pre">_get_acl_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'authz':</span> <span class="pre">&lt;function</span> <span class="pre">_get_authz_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'file_name':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_name_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'file_size':</span> <span class="pre">&lt;function</span> <span class="pre">_get_file_size_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'guid':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'md5':</span> <span class="pre">&lt;function</span> <span class="pre">_get_md5_from_row&gt;</span></em>, <em class="sig-param"><span class="pre">'urls':</span> <span class="pre">&lt;function</span> <span class="pre">_get_urls_from_row&gt;}</span></em>, <em class="sig-param"><span class="pre">manifest_file_delimiter=None</span></em>, <em class="sig-param"><span class="pre">output_filename='verify-manifest-errors-1632510633.7002015.log'</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/indexing/verify_manifest.html#async_verify_object_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.indexing.verify_manifest.async_verify_object_manifest" title="Permalink to this definition">¶</a></dt>
 <dd><p>Verify all file object records into a manifest csv</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters</dt>

docs/_build/html/tools/metadata.html

@@ -102,7 +102,7 @@ <h1>Metadata Tools<a class="headerlink" href="#metadata-tools" title="Permalink
 
 <dl class="py function">
 <dt id="gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest">
-<em class="property"><span class="pre">async</span> </em><code class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></code><code class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="pre">commons_url</span></em>, <em class="sig-param"><span class="pre">manifest_file</span></em>, <em class="sig-param"><span class="pre">metadata_source</span></em>, <em class="sig-param"><span class="pre">auth=None</span></em>, <em class="sig-param"><span class="pre">max_concurrent_requests=24</span></em>, <em class="sig-param"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></em>, <em class="sig-param"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></em>, <em class="sig-param"><span class="pre">manifest_file_delimiter=None</span></em>, <em class="sig-param"><span class="pre">output_filename='ingest-metadata-manifest-errors-1630075392.1731167.log'</span></em>, <em class="sig-param"><span class="pre">get_guid_from_file=True</span></em>, <em class="sig-param"><span class="pre">metadata_type=None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
+<em class="property"><span class="pre">async</span> </em><code class="sig-prename descclassname"><span class="pre">gen3.tools.metadata.ingest_manifest.</span></code><code class="sig-name descname"><span class="pre">async_ingest_metadata_manifest</span></code><span class="sig-paren">(</span><em class="sig-param"><span class="pre">commons_url</span></em>, <em class="sig-param"><span class="pre">manifest_file</span></em>, <em class="sig-param"><span class="pre">metadata_source</span></em>, <em class="sig-param"><span class="pre">auth=None</span></em>, <em class="sig-param"><span class="pre">max_concurrent_requests=24</span></em>, <em class="sig-param"><span class="pre">manifest_row_parsers={'guid_for_row':</span> <span class="pre">&lt;function</span> <span class="pre">_get_guid_for_row&gt;</span></em>, <em class="sig-param"><span class="pre">'indexed_file_object_guid':</span> <span class="pre">&lt;function</span> <span class="pre">_query_for_associated_indexd_record_guid&gt;}</span></em>, <em class="sig-param"><span class="pre">manifest_file_delimiter=None</span></em>, <em class="sig-param"><span class="pre">output_filename='ingest-metadata-manifest-errors-1632510634.1446126.log'</span></em>, <em class="sig-param"><span class="pre">get_guid_from_file=True</span></em>, <em class="sig-param"><span class="pre">metadata_type=None</span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/metadata/ingest_manifest.html#async_ingest_metadata_manifest"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest" title="Permalink to this definition">¶</a></dt>
 <dd><p>Ingest all metadata records into a manifest csv</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters</dt>

gen3/cli/configure.py

@@ -6,16 +6,31 @@
 @click.command()
 @click.option("--profile", help="name of the profile to name for this credentials")
 @click.option("--cred", help="path to the credentials.json")
-def configure(profile, cred):
+@click.option(
+    "--apiendpoint", default="", help="[optional] API endpoint of the commons"
+)
+@click.option(
+    "--use-shepherd",
+    default="",
+    help="[optional] true/false -- to enable/disable the gen3Sdk to upload using the Gen3 Object Management API (if available)",
+)
+@click.option(
+    "--min-shepherd-version",
+    default="",
+    help="[optional] Specify the minimum version of the Gen3 Object Management API ",
+)
+def configure(profile, cred, apiendpoint, use_shepherd, min_shepherd_version):
     """Command to configure multiple profiles with corresponding credentials
 
-    ./gen3 configure --profile=<profile-name> --cred=<path-to-credential/cred.json>
+    ./gen3 configure --profile=<profile-name> --cred=<path-to-credential/cred.json> --apiendpoint=https://example.com --use-sheperd=<true/false> --min-shepherd-version=<version_number>
     """
 
     logging.info(f"Configuring profile [ {profile} ] with credentials at {cred}")
 
     try:
-        profile_title, new_lines = config_tool.get_profile_from_creds(profile, cred)
+        profile_title, new_lines = config_tool.get_profile_from_creds(
+            profile, cred, apiendpoint, use_shepherd, min_shepherd_version
+        )
         lines = config_tool.get_current_config_lines()
         config_tool.update_config_lines(lines, profile_title, new_lines)
     except Exception as e:

gen3/configure.py

@@ -29,26 +29,30 @@
 CONFIG_FILE_PATH = expanduser("~/.gen3/config")
 
 
-def get_profile_from_creds(profile, cred):
+def get_profile_from_creds(
+    profile, cred, api_endpoint="", use_sheperd="", min_sheperd_version=""
+):
     with open(expanduser(cred)) as f:
         creds_from_json = json.load(f)
         credentials = OrderedDict()
         credentials["key_id"] = creds_from_json["key_id"]
         credentials["api_key"] = creds_from_json["api_key"]
-        credentials["api_endpoint"] = auth_tool.endpoint_from_token(
-            credentials["api_key"]
+        credentials["api_endpoint"] = (
+            api_endpoint
+            if api_endpoint != ""
+            else auth_tool.endpoint_from_token(credentials["api_key"])
         )
         credentials["access_key"] = auth_tool.get_access_token_with_key(credentials)
-        credentials["use_shepherd"] = ""
-        credentials["min_shepherd_version"] = ""
+        credentials["use_shepherd"] = use_sheperd
+        credentials["min_shepherd_version"] = min_sheperd_version
     profile_line = "[" + profile + "]\n"
     new_lines = [key + "=" + value + "\n" for key, value in credentials.items()]
     new_lines.append("\n")  # Adds an empty line between two profiles.
     return profile_line, new_lines
 
 
 def get_current_config_lines():
-    """ Read lines from the config file if exists in ~/.gen3 folder, else create new config file """
+    """Read lines from the config file if exists in ~/.gen3 folder, else create new config file"""
     try:
         with open(CONFIG_FILE_PATH) as configFile:
             logging.info(f"Reading existing config file at {CONFIG_FILE_PATH}")
@@ -60,17 +64,18 @@ def get_current_config_lines():
 
 
 def update_config_lines(lines, profile_title, new_lines):
-    """ Update config file contents with the new profile values """
+    """Update config file contents with the new profile values"""
 
     if profile_title in lines:
         profile_line_index = lines.index(profile_title)
         next_profile_index = len(lines)
-        for i in range(profile_line_index, len(lines)):
+        for i in range(profile_line_index + 1, len(lines)):
             if lines[i][0] == "[":
                 next_profile_index = i
                 break
         del lines[profile_line_index:next_profile_index]
+    lines.append(profile_title)
+    lines += new_lines
 
-    with open(CONFIG_FILE_PATH, "a+") as configFile:
-        configFile.write(profile_title)
-        configFile.writelines(new_lines)
+    with open(CONFIG_FILE_PATH, "w+") as configFile:
+        configFile.writelines(lines)

tests/test_configure.py

@@ -42,8 +42,6 @@ def mock_access_key(_):
 def test_get_profile_from_creds(monkeypatch):
     test_file_name = str(uuid.uuid4()) + ".json"
     try:
-        profile = "DummyProfile"
-        creds = {"key_id": "1234", "api_key": "abc"}
         with open(test_file_name, "w+") as cred_file:
             json.dump(creds, cred_file)