feat(cli): add initial CLI support for object manifests (#97)

* feat(cli): add initial CLI support for object manifests

* Apply automatic documentation changes

* Apply automatic documentation changes

* feat(objects): add more options for click for manifests

* Apply automatic documentation changes

* Update gen3/tools/indexing/verify_manifest.py

Co-authored-by: Pauline Ribeyre <[email protected]>

* Apply automatic documentation changes

* Apply automatic documentation changes

* fix(jenkins): update to branch of jenkins lib to test fix

* Apply automatic documentation changes

* Update Jenkinsfile

* feat(delete): add support for deleting GUIDs

* Apply automatic documentation changes

* fix(download_manifest): make download manifest deterministic, sort array-like columns

* fix(python): use python3

* Apply automatic documentation changes

* chore(deps): install fastavro

* Apply automatic documentation changes

* Update gen3/cli/objects.py

Co-authored-by: Pauline Ribeyre <[email protected]>

* Update gen3/cli/objects.py

Co-authored-by: Pauline Ribeyre <[email protected]>

* Apply automatic documentation changes

* Update gen3/cli/objects.py

Co-authored-by: Pauline Ribeyre <[email protected]>

* Apply automatic documentation changes

* fix(review): adjust based on review, cleanup logging, more performance parsing of GUID, remove unnecessary var

* Apply automatic documentation changes

* feat(objects): move delete guids out of CLI into SDK so it can be used by both

* Apply automatic documentation changes

Co-authored-by: Alexander VT <[email protected]>
Co-authored-by: Pauline Ribeyre <[email protected]>

docs/_build/html/_modules/gen3/index.html

@@ -773,7 +773,8 @@ <h1>Source code for gen3.index</h1><div class="highlight"><pre>
 
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">rec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">guid</span><span class="p">)</span>
-        <span class="n">rec</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
+        <span class="k">if</span> <span class="n">rec</span><span class="p">:</span>
+            <span class="n">rec</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
         <span class="k">return</span> <span class="n">rec</span></div>
 
     <span class="c1">### Query Requests</span>

docs/_build/html/_modules/gen3/tools/indexing/download_manifest.html

@@ -325,12 +325,21 @@ <h1>Source code for gen3.tools.indexing.download_manifest</h1><div class="highli
                     <span class="n">manifest_row</span> <span class="o">=</span> <span class="p">[</span>
                         <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;did&quot;</span><span class="p">),</span>
                         <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                            <span class="p">[</span><span class="n">url</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;%20&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;urls&quot;</span><span class="p">)]</span>
+                            <span class="nb">sorted</span><span class="p">(</span>
+                                <span class="p">[</span><span class="n">url</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;%20&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">url</span> <span class="ow">in</span> <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;urls&quot;</span><span class="p">)]</span>
+                            <span class="p">)</span>
                         <span class="p">),</span>
                         <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
-                            <span class="p">[</span><span class="n">auth</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;%20&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">auth</span> <span class="ow">in</span> <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;authz&quot;</span><span class="p">)]</span>
+                            <span class="nb">sorted</span><span class="p">(</span>
+                                <span class="p">[</span>
+                                    <span class="n">auth</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;%20&quot;</span><span class="p">)</span>
+                                    <span class="k">for</span> <span class="n">auth</span> <span class="ow">in</span> <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;authz&quot;</span><span class="p">)</span>
+                                <span class="p">]</span>
+                            <span class="p">)</span>
+                        <span class="p">),</span>
+                        <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
+                            <span class="nb">sorted</span><span class="p">([</span><span class="n">a</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;%20&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;acl&quot;</span><span class="p">)])</span>
                         <span class="p">),</span>
-                        <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">a</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;%20&quot;</span><span class="p">)</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;acl&quot;</span><span class="p">)]),</span>
                         <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;hashes&quot;</span><span class="p">,</span> <span class="p">{})</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;md5&quot;</span><span class="p">),</span>
                         <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;size&quot;</span><span class="p">),</span>
                         <span class="n">record</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;file_name&quot;</span><span class="p">),</span>

docs/_build/html/_modules/gen3/tools/indexing/index_manifest.html

@@ -74,6 +74,7 @@ <h1>Source code for gen3.tools.indexing.index_manifest</h1><div class="highlight
 <span class="kn">import</span> <span class="nn">sys</span>
 <span class="kn">import</span> <span class="nn">traceback</span>
 
+<span class="kn">from</span> <span class="nn">gen3.index</span> <span class="kn">import</span> <span class="n">Gen3Index</span>
 <span class="kn">from</span> <span class="nn">gen3.auth</span> <span class="kn">import</span> <span class="n">Gen3Auth</span>
 <span class="kn">from</span> <span class="nn">gen3.tools.indexing.manifest_columns</span> <span class="kn">import</span> <span class="p">(</span>
     <span class="n">GUID_COLUMN_NAMES</span><span class="p">,</span>
@@ -680,6 +681,51 @@ <h1>Source code for gen3.tools.indexing.index_manifest</h1><div class="highlight
     <span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span><span class="n">filename</span><span class="o">=</span><span class="s2">&quot;index_object_manifest.log&quot;</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="n">logging</span><span class="o">.</span><span class="n">DEBUG</span><span class="p">)</span>
     <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">()</span><span class="o">.</span><span class="n">addHandler</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">StreamHandler</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">stdout</span><span class="p">))</span>
     <span class="n">index_object_manifest_cli</span><span class="p">()</span>
+
+
+<div class="viewcode-block" id="delete_all_guids"><a class="viewcode-back" href="../../../../tools/indexing.html#gen3.tools.indexing.index_manifest.delete_all_guids">[docs]</a><span class="k">def</span> <span class="nf">delete_all_guids</span><span class="p">(</span><span class="n">auth</span><span class="p">,</span> <span class="n">file</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Delete all GUIDs specified in the object manifest.</span>
+
+<span class="sd">    WARNING: THIS COMPLETELY REMOVES INDEX RECORDS. USE THIS ONLY IF YOU KNOW</span>
+<span class="sd">             THE IMPLICATIONS.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">index</span> <span class="o">=</span> <span class="n">Gen3Index</span><span class="p">(</span><span class="n">auth</span><span class="o">.</span><span class="n">endpoint</span><span class="p">,</span> <span class="n">auth_provider</span><span class="o">=</span><span class="n">auth</span><span class="p">)</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">is_healthy</span><span class="p">():</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;uh oh! The indexing service is not healthy in the commons </span><span class="si">{</span><span class="n">auth</span><span class="o">.</span><span class="n">endpoint</span><span class="si">}</span><span class="s2">&quot;</span>
+        <span class="p">)</span>
+        <span class="n">exit</span><span class="p">()</span>
+
+    <span class="c1"># try to get delimeter based on file ext</span>
+    <span class="n">file_ext</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">splitext</span><span class="p">(</span><span class="n">file</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">file_ext</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;.tsv&quot;</span><span class="p">:</span>
+        <span class="n">manifest_file_delimiter</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="se">\t</span><span class="s2">&quot;</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="c1"># default, assume CSV</span>
+        <span class="n">manifest_file_delimiter</span> <span class="o">=</span> <span class="s2">&quot;,&quot;</span>
+
+    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">file</span><span class="p">,</span> <span class="s2">&quot;r&quot;</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;utf-8-sig&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">input_file</span><span class="p">:</span>
+        <span class="n">csvReader</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">DictReader</span><span class="p">(</span><span class="n">input_file</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="n">manifest_file_delimiter</span><span class="p">)</span>
+        <span class="n">fieldnames</span> <span class="o">=</span> <span class="n">csvReader</span><span class="o">.</span><span class="n">fieldnames</span>
+
+        <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;got fieldnames from </span><span class="si">{</span><span class="n">file</span><span class="si">}</span><span class="s2">: </span><span class="si">{</span><span class="n">fieldnames</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+
+        <span class="c1"># figure out which permutation of the name GUID is being used 1 time</span>
+        <span class="c1"># then use it for all future rows</span>
+        <span class="n">guid_name</span> <span class="o">=</span> <span class="s2">&quot;guid&quot;</span>
+        <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="p">[</span><span class="s2">&quot;guid&quot;</span><span class="p">,</span> <span class="s2">&quot;GUID&quot;</span><span class="p">,</span> <span class="s2">&quot;did&quot;</span><span class="p">,</span> <span class="s2">&quot;DID&quot;</span><span class="p">]:</span>
+            <span class="k">if</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">fieldnames</span><span class="p">:</span>
+                <span class="n">guid_name</span> <span class="o">=</span> <span class="n">name</span>
+
+        <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;using </span><span class="si">{</span><span class="n">guid_name</span><span class="si">}</span><span class="s2"> to retrieve GUID to delete...&quot;</span><span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">csvReader</span><span class="p">:</span>
+            <span class="n">guid</span> <span class="o">=</span> <span class="n">row</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">guid_name</span><span class="p">)</span>
+
+            <span class="k">if</span> <span class="n">guid</span><span class="p">:</span>
+                <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;deleting GUID record:</span><span class="si">{</span><span class="n">guid</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+                <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">index</span><span class="o">.</span><span class="n">delete_record</span><span class="p">(</span><span class="n">guid</span><span class="o">=</span><span class="n">guid</span><span class="p">))</span></div>
 </pre></div>
 
           </div>

docs/_build/html/_modules/gen3/tools/indexing/verify_manifest.html

@@ -247,6 +247,8 @@ <h1>Source code for gen3.tools.indexing.verify_manifest</h1><div class="highligh
             <span class="c1"># default, assume CSV</span>
             <span class="n">manifest_file_delimiter</span> <span class="o">=</span> <span class="s2">&quot;,&quot;</span>
 
+    <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;detected </span><span class="si">{</span><span class="n">manifest_file_delimiter</span><span class="si">}</span><span class="s2"> as delimiter between columns&quot;</span><span class="p">)</span>
+
     <span class="k">await</span> <span class="n">_verify_all_index_records_in_file</span><span class="p">(</span>
         <span class="n">commons_url</span><span class="p">,</span>
         <span class="n">manifest_file</span><span class="p">,</span>

docs/_build/html/genindex.html

@@ -144,6 +144,8 @@ <h2 id="D">D</h2>
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%; vertical-align: top;"><ul>
       <li><a href="metadata.html#gen3.metadata.Gen3Metadata.delete">delete() (gen3.metadata.Gen3Metadata method)</a>
+</li>
+      <li><a href="tools/indexing.html#gen3.tools.indexing.index_manifest.delete_all_guids">delete_all_guids() (in module gen3.tools.indexing.index_manifest)</a>
 </li>
       <li><a href="metadata.html#gen3.metadata.Gen3Metadata.delete_index_key_path">delete_index_key_path() (gen3.metadata.Gen3Metadata method)</a>
 </li>