From 0d3b914712bca3db00061a735a8d73794c876f47 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Tue, 2 Nov 2021 09:18:28 -0500 Subject: [PATCH] Print output when unable to parse JSON (#102) * Print output when unable to parse JSON * Apply automatic documentation changes Co-authored-by: paulineribeyre --- docs/_build/doctrees/environment.pickle | Bin 225740 -> 226070 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 93370 -> 93366 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 34277 -> 34281 bytes .../_build/html/_modules/gen3/submission.html | 12 ++++++++++-- docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- gen3/submission.py | 12 ++++++++++-- 8 files changed, 23 insertions(+), 7 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index dd3923a5d50456bf128824f61f48c7fcf5fbe488..14d07602a67d81a4bebcf711c3bb2d299313d2c8 100644 GIT binary patch delta 4365 zcmcIoYjjlA73SPYW|9!|kUYryF_S=ELjVaOOfm`YJSLMchzdwDkRi<_@0kfA1gKUt z=8^|{$RU%!giKJBXal28>(aGqTXD6h6TeH<61+`dg<_5FPjAT=5gXNK?)<%aE*kQMH z?siB4JDQp;$nqVB6k@e+YiY7to7!4&Pi)(U4g$<}+YcO4827he#)V>+6?c8&=|cwW z+ay(&rPD)FzY_M>$9&gyRwW)2X3bh#_Azpg+)5LVPjkE9c~u;l=A2(uiDT2;*zaeH z&S`G4Wr;XC&7Ix%km#D`K3x+l4o`Ex{rO^X+{dNGe0Mu5T%7Q4-XgnS6y4LD>e^P2 z)ElE&c>Rv~Qn*&TpwZI0cmXR47&NgErkPE~AYK!%W1%peIer~~#@hDXjsYGU+m7&W z^ZLqrE@;=Jl3^IsXvjF+QOCjIj2!rXSvdGxV;$ka{imfarwz}bu_a$q1o;)$06HkX{ z!js_fyFJ-U@&ybf5_T@V1}_)eu5aWg3J{fI9PfPzRfRy+pCsg zcjm$eWhVGwp#~~eOf6?|{73q%pezkm&Vvq%6x!6fz1iMiVUIGekX-T!$X>F`Gr2O3 zK!=ezKLs0WG@iFss|ga3p4)4llnGdOP$se~dI{{)7r7jJ zV{wqPHqw((T}9x{RdNECjJx2=HFC0#)_lH}m9&!Z=cZENZ9$Gz7Z+AvTv&y1VKv5u zWrqvPs4{>jgm!M2s9m{=^&Pv-o6S2dKZO6gSEz5{4w)5y=q4a%y=FgiG{8OJTgzh` z;I3vk=5vN)fzEI&-WiUCJ;Sl+XE-)&W;hlAFg3?zz3bK|zi| z{l-b9oGn}tqdb`pWQUM3Pp$b?fjB*(4Ie4VsONLBfe@ExU2{kXju%r`F&Xy^cDZJg z3D3gE<_8eBXVbwK6veCoJv_%Ao?{QMeh)8x53hX>FMSU$R1Ysx4-c*f-^o=^mlKSq zueS{Gz1~G+1QgP0#BymR;vuOD@uIX&$={kPsFw1mtCoZ+Uk}1}dITB|twDts59txV zKD1g^dk`M&n+NrMQ7E+cl_I{E(I24zoO7I-HMp#TSRyAm+kyC4VT7VgMPj zjTT#7o88uKxB2qmdn0TQ-i1gsxC}bPB*${Zha5$S4o4wz!aI&q#IPgfDn5$Ns9?|? z2fL1Bq4dO&6-qAk!h)eZT7HIv$pvsi_RwlnXdlueP7PHf-sQBOGP;lI5YN*p+59(f zVK^3so6=QsqE2}D=t@*LakK>S@=@j_b$AgO1=Fw|5rzv9-yU9t80pjz7gRedRD3{Q zSAlsnpT1WjgwVSsf)Xw{1C@N|8@MFJLikubt?kTy{W z5qkoC4;&X(F=3>seA8-QMYz<6(kQs*)}nplgwOtdrW$(x`YMxO6<(Yumwj*M2G(vZ z=uawT(N-?1J6VW6j-K3sc=u!rV#_lRBA$DO#UJvl&%T{2w>-;)XP!079=hQCGbzyg z_*^iaQsI^Gz$ufGU+3Li`wuE=2|1qz-ZS2O%hU>ofNwke_NKb zpWxaCOS8phsq3_CZ|iKp18D7f5+$D+2jTq*vip72Bo%#~UQc|2I|h;{YenNr9TjYq zW0R5?$0y)2e$t$oli297+RQe;>b?KIW$(mq1UbPoRU{5i+hgJh44X)*97TT>nJZJe z8b38+B=M*lv4Q39U9f>D()qlTQsA;VRS?JMrY;hn$i|$+gQ>tz3GMgC;20f14cRCP zh$AL|Dbou!5-SVug`3;wpeOG?y2ww9_@t8J;J}fD05vs@2#RYGi#iO}#D%_yys6sp>1(!4rX6dA* z9;}a(Zt`I2POsfVq+})gA%J78edk<39HnXRlb8S|qEi>pa+GdrE1b)WH9XYl^P7dM~I1+mZkadD20z=!N9qZ$f6x zGBgrH4{a|NQiBf>!x`L@(%|OtIkdl2hzR8x9%6Wd+0>K@nPFV=CNX?Gqlr^EI`|?n ze8EI&G|rIit`fs7W_O@WNG|1eeMPu3Q^OSQa}K{4qTVZB5)s_a5{wc=>;Wm4NdD$ z>4l*0DX1t4l(K{0>4h||s+NzG>%;CIG%%D3S>G44vdMv@%i;_yNCFMk385&yQ7UB1 zy9q&}WjlM5eL=Gu33tq8LIN(`WxgY$Xv$o?V(WD*I`1#agoXtwc57Ir4CvS!1(zR* z^qM*Z`({4HrA)YRa=uJvBbiJ?_X(l++lZM+(e#I}V5elxE~TS`bZ_fBLjN){?!BW_ zc#M3SONsEaClXPOjek2f)cm;Gdv~M48<)T)x5TH;16Gr*t<&6S(Qa?U&s@<>FSIL? zh?`z-SES(**`bK#gIQE?S W*dcZ)s1Y9#E&E%-=CIqhY5xUVd&qVG delta 4208 zcmZ`+d32Li7Uz8lNoh+LyKl55X=zKFw#d>YbYtz3Zb4x{fzZa%>46v_uhB+ z_ukEyfxj+(|Fgx@PsJaWKhoG_v;zYvVA5ral7;y`kE|znj1}~#?}^VpJk7r>@;>b zNy3-h;jPKJKYY>24{{3@Y%2a3xqGP(@MB)?^R4`-hs#TN=XEt-%qh3^jWj(P6*%?c1y5605;daWr! zphlC~U~IWRleSLNw%^*vkHXtYQL;XM1ftc^TAB}44Dl#>IwmR81nG8=XPdQJ+M9qj zbpo+Nzgn%N>7gFQ5SPb z>FNJF9j{|%D`nwuE=k@+;O24_j4iW(DOCj-_pgMelx8@%Oa+(H6!3mdB-~9_LSkwy zOr}vi1&pM&LRg9lG-(InPbsO8S|I797Wl)xw2iPhLj_+Is^IAIov;CY`={@Pag+_E zo4^l!WM!COU%CpeSE=EPi~~m6a6VRsK<@&dMk#s^_#8^nJHT(G6zw;^g;KP&{2oft zbLZPAMbDf+L@C-(UZ4~`YJP}P4(IPR=^RuPuLk?-94sRaj%4d0xkL@$XU{-lUL1U~ z;uE-+r-qO3e`^a(L{J)A%pHwJll5V1N+o{^MP1fPc6;ecGo&lckFGR3y3*X}N;9J? z&5N!M_}j`QYE~;Nj@89VV%4xhSPc$m+NvxLQwfH>YrcltSuO69*b5(Kt6@j(47|Fw z#eEVHFqNkVYk?Y6`7>~FMI035)!@k_!ed%JY+n-$LHQvtr=5XqIc7Ymy|5-%4X+oV zk8-IMUln%3A9a#mSJ4a{DqIbB*5~4>MZ$;0de~E>hO(0Be45+OT0$F5OQUV>vKj^d z#+JG!OTCdk7}hoED25K_b|BXX7&x0h_xdBCYHw zUd#N|@MZZ~cu%W#x+<0sB!VHWEDR1m6$Q&T1v`@}D+v5{y;R2as(tXqMyZSg)t_ym zjV>qbMwN5yu#lrY$Axws7ut7RXy*Cf()IlR zCy4q+=8#&k>o^3um>(9r9X_p%Qm`jyciyS3myto({os_3R4qU14Be{rB_qxn!oW^qp(TxU;|0?nj24*-xd&iNm?Q>%43&ZGShbu$xub&3fO>THnn& z-_07|&Dzw>n$*p@-i>c{{d1)Rv*|fnjCkef3Ni{ZVFO~RunzH} zqWy5Ece&W3CrYo=mHjk3b3YC4>R&43kqV2wCk#4Yk?C(O+5Flk_0M=Niv^Ym;2f=0?P*p88M$yV+IvqCuYzO$H9!gSk0 zcLfvh%}AM)fiK(`T_w(jZ~@-vM@OsC&!?kRi0QUc#7>(Y@ej5d#8`W&WEv0Iwrm)& zhoErUULple@&rvssdkJe^vqZaKELU)bz~g=IaZEXbZRG}o#EY6HAq)GjEJu}N)Z3; z*sNfKs1y<>avOTn*|RJok_!>|(-yoGU|N z!3(<)?Juk*4*2MW9K^uMwTQaOV#MCbH3~X5@!PI=Dix}p^>Lr81LRX{P(eGTMchA? zkNE0TA>!?+JUoM}^KqDNCBr6$&mj&u&z^VC=~X_LNCutFbKgJ22`bkk734WN85M6; z6E%CyLbTYfBhd;go04Lyjx3YYp(a*U5|u>lt|m()lQ8i=RU{DUt2HDJm$@2`&Xr0M zlR)i+h)wv(T1vm-HTA|OquIEt!&ukaQI7${npzSnMU8^M-WbXKp-Pf~z9wslTX0uL zVkNCGe8r)QOj2$_9Oc-^%Vm*5ypiavF_{f!kLrCt-LiBvxp%}16(kCyJ+5a34C+aO zltpg^iIga9wTBw6BGITDw1w91%G^R^OWA-E;$ha1!0{vE_D&KVOXr%vk}1RgFq$9F zy%DhwH6)`@Al7g>RTi(BNrWW44{p_kpeNT4o#c%?HkyPeIC3IJPKTUmm?qH@m3M)} zdA6$tV&dtx6jbN_FQuvLFdEE8EBlEPZ@fg3iA{XsWsig4X(o8*Wem-X6W|+hDXI`B ziN__;iB;OfA}998CT@3P={A?eNrWW|`n!)4t!-~4#}A8%*GRaWip1%wXgMtIxJH(7 z7aLr^x<=|U2RJT8TrA@P#EU-MPFI4A`?Y*t=0e%ERm33n6>!PQI-+}$JYyAKEa4&- zee1hmf$nLdJ0}(wa*?7okPGm$@=Q8ObZ>}(g`7ryk?5|_{ftG|i0;p5E2!X>ElF%^ z=j01@H;8VIdJcuPol5cTLe5Wqi|F1JFBfvL9C#`7Ea*6X)j8Yo66Jm7{ zm&k^o5F=i`O`_?=qxwP|*oWiEg!pC=r^e-a5vP*$=tj7BcP+OF#b_QXdS)`mPl#p3 zoC@Xd6k|Q^SDtPBfmH7H4E{M)T*PxGVPZb#wf7mKhks(*YHrZ?x zafqZt7A2=+OeF2HXg1Jr!X$La!qvO*6DvoW2{?y3Q*3@disp1fCbjadqXp=1Lze4zohX4Qo diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index 0d8f735805583a736a3caab861e74d2c16716818..d887e1ebaa6d67186aa825606e4ad9ec52b5692d 100644 GIT binary patch delta 137 zcmdmWlXcro)`lsJm5ioas~A(28Q*Mwpv-uikL6uQk234b8PJX)jlnnORDH&ot5EbY~Z!%=$um%7Lu_e#| delta 141 zcmdmXlXcfk)`lsJm5i2Ks~A(28Q*Sypv-uik>A+F)WXETQqRoX(9qo6Wcx`~#wkpU z7q`c0Fz&~c)z)HUVrR6UK0%dHTNX{LUQT|x`jiaz*eMxeJrE@rrmr((b;FDaq28{85tNR9}JY++@fyF$Zu+4U}Ru!s%L3zW@2PuwE4Wo z2PVe)%?Gp{Fytnm*Ra{_rTdzN(PMJ2hKVeiR=u43boD72?6Ffa#CpIgFbqFoBpVC> DAz3Mk delta 134 zcmaFa&GfXJiKT&M>aC3|aq28{7#SEQY0GVHQMYB}Gd3}`FflaHGc+_ZF*4hHLE{4x vW8LP1+71{}0=nv~jP8?rHB4ksHRjrNV2_=WA=U#{fNts;BiUd84~8V^ diff --git a/docs/_build/html/_modules/gen3/submission.html b/docs/_build/html/_modules/gen3/submission.html index 4b4f133e7..0f7b5a7ba 100644 --- a/docs/_build/html/_modules/gen3/submission.html +++ b/docs/_build/html/_modules/gen3/submission.html @@ -379,7 +379,11 @@

Source code for gen3.submission

         output = requests.get(api_url, auth=self._auth_provider).text
         if filename is None:
             if fileformat == "json":
-                output = json.loads(output)
+                try:
+                    output = json.loads(output)
+                except ValueError as e:
+                    print(f"Output: {output}\nUnable to parse JSON: {e}")
+                    raise
             return output
         else:
             self.__export_file(filename, output)
@@ -411,7 +415,11 @@ 
Source code for gen3.submission
         output = requests.get(api_url, auth=self._auth_provider).text
         if filename is None:
             if fileformat == "json":
-                output = json.loads(output)
+                try:
+                    output = json.loads(output)
+                except ValueError as e:
+                    print(f"Output: {output}\nUnable to parse JSON: {e}")
+                    raise
             return output
         else:
             self.__export_file(filename, output)
diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js
index 3c3055394..5a5d6abf6 100644
--- a/docs/_build/html/searchindex.js
+++ b/docs/_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0934136":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"112426":10,"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1633103143":[],"1633373719":[],"1633449426":[],"1633449427":[],"1633454066":[],"1633454067":[],"1633458409":9,"1633458410":10,"1731167":[],"197634":[],"2280114":[],"2343018":[],"255e396f":9,"2889535":[],"333":5,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5859108":[],"5964222":[],"622057":[],"641011":[],"6542356":[],"6572418":[],"6711774":9,"6f90":7,"7461076":[],"7649791":[],"772628":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9781935":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],use_agg_md:5,used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}})
\ No newline at end of file
+Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0934136":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"112426":[],"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1633103143":[],"1633373719":[],"1633449426":[],"1633449427":[],"1633454066":[],"1633454067":[],"1633458409":[],"1633458410":[],"1635802075":[9,10],"1731167":[],"197634":[],"2280114":[],"2343018":[],"255e396f":9,"2889535":[],"333":5,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"480863":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5859108":[],"5964222":[],"622057":[],"641011":[],"6542356":[],"6572418":[],"6711774":[],"6f90":7,"7461076":[],"7649791":[],"772628":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"9159722":[],"9335642":[],"934012":[],"9364202":10,"93d9af72":9,"9644923":[],"9781935":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],use_agg_md:5,used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}})
\ No newline at end of file
diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html
index 988bf898f..fd59dc846 100644
--- a/docs/_build/html/tools/indexing.html
+++ b/docs/_build/html/tools/indexing.html
@@ -365,7 +365,7 @@ 
Indexing Tools
 

-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1633458409.6711774.log')[source]

+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1635802075.480863.log')[source]
 
Verify all file object records into a manifest csv

 

 
Parameters

diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html
index 29a246fdf..8ef51ad0d 100644
--- a/docs/_build/html/tools/metadata.html
+++ b/docs/_build/html/tools/metadata.html
@@ -102,7 +102,7 @@ 
Metadata Tools
 

-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1633458410.112426.log', get_guid_from_file=True, metadata_type=None)[source]

+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1635802075.9364202.log', get_guid_from_file=True, metadata_type=None)[source]
 
Ingest all metadata records into a manifest csv

 

 
Parameters

diff --git a/gen3/submission.py b/gen3/submission.py
index 16ddf66bd..166005289 100755
--- a/gen3/submission.py
+++ b/gen3/submission.py
@@ -346,7 +346,11 @@ def export_record(self, program, project, uuid, fileformat, filename=None):
         output = requests.get(api_url, auth=self._auth_provider).text
         if filename is None:
             if fileformat == "json":
-                output = json.loads(output)
+                try:
+                    output = json.loads(output)
+                except ValueError as e:
+                    print(f"Output: {output}\nUnable to parse JSON: {e}")
+                    raise
             return output
         else:
             self.__export_file(filename, output)
@@ -378,7 +382,11 @@ def export_node(self, program, project, node_type, fileformat, filename=None):
         output = requests.get(api_url, auth=self._auth_provider).text
         if filename is None:
             if fileformat == "json":
-                output = json.loads(output)
+                try:
+                    output = json.loads(output)
+                except ValueError as e:
+                    print(f"Output: {output}\nUnable to parse JSON: {e}")
+                    raise
             return output
         else:
             self.__export_file(filename, output)