Skip to content

Commit

Permalink
Merge pull request #90 from uc-cdis/xritter1-patch-1
Browse files Browse the repository at this point in the history 
empty PR to test for ext. PR docs_build failure
  • Loading branch information
@xritter1
xritter1 authored May 6, 2021
2 parents 9bbfa7b + dae2682 commit d82eeea
Show file tree
Hide file tree
Showing 9 changed files with 37 additions and 11 deletions.
Binary file modified docs/_build/doctrees/environment.pickle
View file
Binary file not shown.
Binary file modified docs/_build/doctrees/tools/indexing.doctree
View file
Binary file not shown.
Binary file modified docs/_build/doctrees/tools/metadata.doctree
View file
Binary file not shown.
33 changes: 26 additions & 7 deletions docs/_build/html/_modules/gen3/auth.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,9 @@ <h1>Source code for gen3.auth</h1><div class="highlight"><pre>
<span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">urllib.parse</span> <span class="kn">import</span> <span class="n">urlparse</span>
<span class="kn">import</span> <span class="nn">backoff</span>

<span class="kn">from</span> <span class="nn">gen3.utils</span> <span class="kn">import</span> <span class="n">raise_for_status</span>
<span class="kn">from</span> <span class="nn">gen3.utils</span> <span class="kn">import</span> <span class="n">DEFAULT_BACKOFF_SETTINGS</span><span class="p">,</span> <span class="n">raise_for_status</span>


<span class="k">class</span> <span class="nc">Gen3AuthError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
Expand Down Expand Up @@ -289,7 +290,7 @@ <h1>Source code for gen3.auth</h1><div class="highlight"><pre>
<span class="k">return</span> <span class="n">_response</span>

<div class="viewcode-block" id="Gen3Auth.refresh_access_token"><a class="viewcode-back" href="../../auth.html#gen3.auth.Gen3Auth.refresh_access_token">[docs]</a> <span class="k">def</span> <span class="nf">refresh_access_token</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot; Get a new access token &quot;&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Get a new access token&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_use_wts</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span> <span class="o">=</span> <span class="n">get_access_token_from_wts</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_wts_namespace</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wts_idp</span>
Expand All @@ -300,21 +301,37 @@ <h1>Source code for gen3.auth</h1><div class="highlight"><pre>
<span class="n">cache_file</span> <span class="o">=</span> <span class="n">token_cache_file</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_refresh_token</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_refresh_token</span><span class="p">[</span><span class="s2">&quot;api_key&quot;</span><span class="p">]</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wts_idp</span>
<span class="p">)</span>

<span class="k">try</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_write_to_file</span><span class="p">(</span><span class="n">cache_file</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;Exceeded number of retries, unable to write to cache file.&quot;</span>
<span class="p">)</span>

<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span></div>

<span class="nd">@backoff</span><span class="o">.</span><span class="n">on_exception</span><span class="p">(</span>
<span class="n">wait_gen</span><span class="o">=</span><span class="n">backoff</span><span class="o">.</span><span class="n">expo</span><span class="p">,</span> <span class="n">exception</span><span class="o">=</span><span class="ne">Exception</span><span class="p">,</span> <span class="o">**</span><span class="n">DEFAULT_BACKOFF_SETTINGS</span>
<span class="p">)</span>
<span class="k">def</span> <span class="nf">_write_to_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cache_file</span><span class="p">,</span> <span class="n">content</span><span class="p">):</span>
<span class="c1"># write a temp file, then rename - to avoid</span>
<span class="c1"># simultaneous writes to same file race condition</span>
<span class="n">temp</span> <span class="o">=</span> <span class="n">cache_file</span> <span class="o">+</span> <span class="p">(</span>
<span class="s2">&quot;.tmp_eraseme_</span><span class="si">%d</span><span class="s2">_</span><span class="si">%d</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">random</span><span class="o">.</span><span class="n">randrange</span><span class="p">(</span><span class="mi">100000</span><span class="p">),</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">())</span>
<span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">temp</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span><span class="p">)</span>
<span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
<span class="n">os</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">temp</span><span class="p">,</span> <span class="n">cache_file</span><span class="p">)</span>
<span class="k">except</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">True</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;failed to write token cache file: &quot;</span> <span class="o">+</span> <span class="n">cache_file</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span></div>
<span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>
<span class="k">raise</span> <span class="n">e</span>

<div class="viewcode-block" id="Gen3Auth.get_access_token"><a class="viewcode-back" href="../../auth.html#gen3.auth.Gen3Auth.get_access_token">[docs]</a> <span class="k">def</span> <span class="nf">get_access_token</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot; Get the access token - auto refresh if within 5 minutes of expiration &quot;&quot;&quot;</span>
<span class="sd">&quot;&quot;&quot;Get the access token - auto refresh if within 5 minutes of expiration&quot;&quot;&quot;</span>
<span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span><span class="p">:</span>
<span class="n">cache_file</span> <span class="o">=</span> <span class="n">token_cache_file</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_refresh_token</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_refresh_token</span><span class="p">[</span><span class="s2">&quot;api_key&quot;</span><span class="p">]</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wts_idp</span>
Expand All @@ -324,10 +341,12 @@ <h1>Source code for gen3.auth</h1><div class="highlight"><pre>
<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">cache_file</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_access_token_info</span> <span class="o">=</span> <span class="n">decode_token</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span><span class="p">)</span>
<span class="k">except</span><span class="p">:</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;ignoring invalid token cache: &quot;</span> <span class="o">+</span> <span class="n">cache_file</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span> <span class="o">=</span> <span class="kc">None</span>
<span class="bp">self</span><span class="o">.</span><span class="n">_access_token_info</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">))</span>

<span class="n">need_new_token</span> <span class="o">=</span> <span class="p">(</span>
<span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_access_token</span>
<span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_access_token_info</span>
Expand Down
5 changes: 4 additions & 1 deletion docs/_build/html/_modules/gen3/submission.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ <h1>Source code for gen3.submission</h1><div class="highlight"><pre>
<span class="kn">import</span> <span class="nn">requests</span>
<span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">logging</span>

<span class="kn">from</span> <span class="nn">gen3.utils</span> <span class="kn">import</span> <span class="n">raise_for_status</span>

Expand Down Expand Up @@ -231,8 +232,10 @@ <h1>Source code for gen3.submission</h1><div class="highlight"><pre>

<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">api_url</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{}</span><span class="s2">/api/v0/submission/</span><span class="si">{}</span><span class="s2">/</span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_endpoint</span><span class="p">,</span> <span class="n">program</span><span class="p">,</span> <span class="n">project</span><span class="p">)</span>
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Using the Sheepdog API URL </span><span class="si">{}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">api_url</span><span class="p">))</span>

<span class="n">output</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">api_url</span><span class="p">,</span> <span class="n">auth</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_auth_provider</span><span class="p">,</span> <span class="n">json</span><span class="o">=</span><span class="n">json</span><span class="p">)</span>
<span class="n">raise_for_status</span><span class="p">(</span><span class="n">output</span><span class="p">)</span>
<span class="n">output</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
<span class="k">return</span> <span class="n">output</span><span class="o">.</span><span class="n">json</span><span class="p">()</span></div>

<div class="viewcode-block" id="Gen3Submission.delete_record"><a class="viewcode-back" href="../../submission.html#gen3.submission.Gen3Submission.delete_record">[docs]</a> <span class="k">def</span> <span class="nf">delete_record</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">program</span><span class="p">,</span> <span class="n">project</span><span class="p">,</span> <span class="n">uuid</span><span class="p">):</span>
Expand Down
3 changes: 3 additions & 0 deletions docs/_build/html/_modules/gen3/tools/indexing/index_manifest.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,7 @@ <h1>Source code for gen3.tools.indexing.index_manifest</h1><div class="highlight
<span class="sd"> auth(Gen3Auth): Gen3 auth or tuple with basic auth name and password</span>
<span class="sd"> replace_urls(bool): flag to indicate if replace urls or not</span>
<span class="sd"> manifest_file_delimiter(str): manifest&#39;s delimiter</span>
<span class="sd"> output_filename(str): output file name for manifest</span>

<span class="sd"> Returns:</span>
<span class="sd"> files(list(dict)): list of file info</span>
Expand All @@ -553,6 +554,8 @@ <h1>Source code for gen3.tools.indexing.index_manifest</h1><div class="highlight
<span class="k">if</span> <span class="ow">not</span> <span class="n">commons_url</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="n">service_location</span><span class="p">):</span>
<span class="n">commons_url</span> <span class="o">+=</span> <span class="s2">&quot;/&quot;</span> <span class="o">+</span> <span class="n">service_location</span>

<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">Using URL </span><span class="si">{}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">commons_url</span><span class="p">))</span>

<span class="n">indexclient</span> <span class="o">=</span> <span class="n">client</span><span class="o">.</span><span class="n">IndexClient</span><span class="p">(</span><span class="n">commons_url</span><span class="p">,</span> <span class="s2">&quot;v0&quot;</span><span class="p">,</span> <span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">)</span>

<span class="c1"># if delimter not specified, try to get based on file ext</span>
Expand Down
Loading

0 comments on commit d82eeea

Please sign in to comment.