From af155a9e22157c8eb440abea19e018cba8077aa9 Mon Sep 17 00:00:00 2001 From: SpencerAxelrod Date: Thu, 9 Jun 2022 20:52:27 +0000 Subject: [PATCH] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 306980 -> 307194 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 87834 -> 87834 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 36345 -> 36340 bytes .../gen3/tools/indexing/verify_manifest.html | 11 ++++++++++- docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 7 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index c6eb4fb4f32d3acbff36ab46249d2eff30ea04a9..022ceb53fde37c4de8a9b3e4411882a91a93969c 100644 GIT binary patch delta 5883 zcmbU_X;hPEy1rjR0+F4t2!u67L}U}$6a_&&mnK_-R?T_Bxdag6~dEYMtJ7>KknVk)tirJE{sT9k|rg6H9Zp)x}$uPrm7wxq_i+2MG6lyy}^ zqGZ<{ZqlTM+4s%}B~2ZAzTd&wZfxLhI=GAZ_xS+_ck7LKe#pTM<@}bv<>2oB(8u3) za1TE6;(Hz3Xk8B9<=|d=wDNtq<7}y9W*R@};JkBp@I4MLa!nUM?BHgcJkEDJxWvDG z0gIw$lRI!cDnN38??)UZDMZlN|3o<})@ zc4{z6!{EQ;oq-@dA#;IMGnDX`cz@FSinR}%i}r!MXm5wM>)&bXVgkkuSC#nQQk zV&kARHrKv9>L>=95(3Z}&GXNlZGheJvsmW-Uk+)hsKZ_HGg!*~yv&Tt;0? zIOvjG;IjnppRLH65D0gtS2+S{o2Gyb(-dSHc*grfQIZmVJv|y~lGb76ZztU*{g9m! z0ftN!d_6M@a#AB9ZeB9Hlbi+DQl`Pz$#1}=0^ zrbWWzWEs?J0zi_QMY@=`ih@X}P0fNivm)W!)Hk3Dtt!$sLSDKW{+_lL)aX&QHW=Pd z&w}62jD)rgYH-b1#x@myNHS>>+faNxrRZkjTPa0165m59+L`=aO3@9(|Cv&B^YGtM zif$bKC8g-5;T2K@w2gRwO3^Oi)s&+9gioXttsOsCO37{*%TSU#)*mt=2rOHyhS_sE z;nG4qW=IQl`D*yD%ue`ei3*mk3WOOe(=l7`!tDYz^v`SEMb~PUYxSn;O0Chd-V$BN z({p04rH}M0E+4IDW;33m@b`Sx^RkVH|(`&&90UDGF?r%mL6Pe3O0k7 zFEz^$OAgBmO9{&cO9ab7w{=FgFNga)C+8*1)|j9vcO!T#QA65-PRPpDW2IG4o0kvD z1#0*quM=7eQy^?z8dg3W7U$=~hec|*klzVQH|VhoDj=p<4K<59A$7Ul-U%DQZG{?U zF6pG*kh=vc3f=<$IcoUNf=>9yf)tjuZ&L#xrzjtW^3+gY)R0ft_Ai;Eg`$zuGEq#6 z^zSpkO3JD^3^TH~SwoghA@Ji0aorVZVbHNWzwUJ}2wUOA z$K=*$E4>La-d69|w~NH@R{w$$QP8m31qM%fSeLHZK%i}|3PLicU=SG)vbGHVW1iZ2 za;*nJ!pqvR?n443Q;!X;gW*gy6c#^RM>{}6ScMSL7SV954UnU?LPV>Dh}H`ctr#L& zGek5qh-i$6Xp|Z$%LRJrhP@keSl3~%z0wH(mp(v8TIQS%eI)h`MW zdRiAFTx(r`@PsvA9JLA#+M?kx@8;|n6We6>xE5NQs7?;&-jYLolN<-yn&u;X(UgX8 z=J{OFNd>Ms8aAGnLf96M2}kYc=aGTB3pt`u6)d&I!q^rWymP@t6dB=3Ljbf{J=s{Q zYQ5lvj{7gnMd)VBM5wW4AvD{vsoa(;XR*HT3|XW8kZ>_Vf#~-q&+Ak_a zb{`9*{i0kLI6|CV?a86pJ_uE~f?3VJWC%3P`3Nn|nKE_)AB7{$QSh?a8-<=NnFtG8 z=E9ZpPH?a#9l73?IS8FEW#D@%=hA#RJFy#`!K+sZ$1f#{e3Qt3dno~}RIRx(X4S%K zysI@vWG~xUKX}xt6h-YAKmd4LR*UR)=(`*z98+-qu=BD^ly%xo{Dgi5Cxf=j&NxNa z<)xyw4-Aa~Q2EP1C~A|VVs%>~!c%Q|2%oks5RL9(NdjQ$l}M3&;PAN9opXlzD>5{` za3zJ@hNo9@5GJ-K<8yFRdp5!|?HLIF+@6Op=IUI$Dk`rok+Xa1CucZuEm8PqF6ZuO z#V(LuTZ~3U*E9%^UMr9>QztlnZH}=2BnFUhU51*}>q|t zMB($!T!hJ8bd9^ZsH?uN7(5q#?8+j8kk-8f;rrd25PsXeM#hfYh46V#F6{5|N8#z7 zjR>7@u0yE3S%mQX%~c2|_pTR>*MMiQFZ`-ETV&TUcCmFGZw z-2oRj`=(^`ZDxwOx9Ea)-Fj2B)}wG9cph|u$bO|L+UgMP85fz@(1!j6qQy4GdJfD( zPfG{v;q0&r+_aLkpTqOG23vnYVn0F`?9(<&L zXM=K4Tq}AUnuF66481N2_cP)1`>t^As2_|C`HSK@CicEP7j2i_E=JgRyO0dl-C2#~ z8+TSBJbR}Ep~vtBg!>SZ9&JGQ@{tx{+2c}#osZK)2d`4P%)IpN(ZBOn5RIcaQFg3JdfbggtQPi7Jy7=(@|KNt>5%US4am z?6L5>UD53@f@twQ|JR>s_fKett=yl6=S-oOq}gquRzEWv|J=uw)l};&?Byh6hY%IH zBe;bUHMwI;3MI*2Qu@RZ2IEP9j8f(h_(ww+=@ zg@lsJMs>M%A??|Dsh4=?i%Ir-mve-~+jMG&b zO~pJ?7JkU~u$BD1NJ=j~;rL`uB~l)fxj=`^R|s|EoY@6ro&tldK_ck`SGOsh*rQ6= zui&D{En%PA_^>M|2&PWWo^;v*`{y7rscf=^~@AT5|w+wg-~iwg>*)=@2P>+6t~ITu>0D%pP>MW%OfIocKwKJ@kac z6fTtX+8(BGV_Qg-HWx7XyN zb%?Ox1c&z^b)rT@>=rKKMMwvsCP?A97rTVflU%rH8Og3Sb2yazDg+vCIH9!g6gO=` z>LVOD#nE-)#1W$SH6i>o7YNTg+)(`eDK19T1_=Z6IVbEx`yxj=-9>vP)bI7>P`vZ> z__RtPY$qp&%U#Z-%XZ^5H`vUrPP82=k<`hh^y(0vZj!{1L1A^JB$OTY*k-$W!clUp zQW7Off*|{*DxX@L-E-~N&k9qCsRp}L4LABcK+$!mzd=Oqy;=D~wCczLz4 z*#4PObQ}mzjs{}F25kjzNd^l^k1eD{(peZxk36$yxmKq&X^TsAy5fqOO8TIoa-qao zIz>*Y8gm3J`z%73?;@Sf9Z9ztT%^ln!pe_{%GU28O-^#9&!Di(utI}gs}LH$;(~>& zG0sW&+d-+H@aY+eOxR|YdZj7C>_6@HGHsP9tfbPg*<`AUE7MitM~z}pil1DKaXCz~ zTx+aCPAlwollTd3H;Jn-=1H8@n19BGsj&8-ohk3`wLa-}zm*iC2-h7*rN$nXV= z)IE@N3m)gBQ)KjZGyB28&;4W;_N_+v?sI9#6eebe%xdEblcBn#LK~;U>U9fy&PjuD i@Ue4Je?-Xn2)$>eE}|nP42}7r`1v_$D4n5R>ia*rGhwU% delta 5715 zcmbVQdsLK1mbbb;x}oJw%e#5Dhy+_uL_|T5R?!v^pD{rZq(#0UjnEv&#Aw_((Ix4` z$L;acW5(#5@i99afoZgT0OG58%1e_NMZwH$ayDZ$IXko2&7Q=)Rn;^)XLiq?{pWX| zb*paOTUForejM@t?IZujThR$ZNu7C%)go-Js5DwF-e!}jvUd7bqq(H4#9U&vsA}s< zw^r2FR+y?FJSKFR8*{K)+{azqw$&MHcDTk?RFxTjQBhTnKI5!>du^@NqA-<;#!Y6c z#f@2Pt7DSoyE|kPCM?drclOL*WA!{#aeFgztlNz-}R9u6)67b`>;Jz zHYB~m_DcOq;7j?sCBMpQU`kAc3)B3eZblK-d}ziH+?s#uEja4Q z>{sD>D(ccUf<7%4zDX;Bjx;SqydDPc&dGu-w6$%}f@gXmJ3!)L`Gg7V@Q4kRqX$Q9 zp&UIlVi)D;fe}BW9PK>ukCdamCVodb+EwCDl%xG4ssseuAz~2a=(!NJl%tgulPO0l zBW4Pe9fHwxHR-VZH9dyFP@Wd%&U3)>4OO@+Mo1{u!hd8q;M{8(SiU+GrmdQTJG2v; zi~JyJSpf9SZ`n=PYf&1j%rzCpTI+gie7;EUg}v5%mK3El5lU$ql+q+9r72KK6QGnX zpHjMbO6ft}Yc*OtE2=hC)|DCQMaQOKGw?mrvRz^u#P)`53)>O48EhYhY}2wQ$#B*F zuqx*X{JgLV&gnP9;36%gEp$Ljekw#2rD2^lP`@Y_-Y?L?Ul%!mWK?0zqhV=oF5Jz~ z!i8K1t-pRNe6yq+g63(VZm9#Z^k(dZji6qrg_+A7(3Yuzh>Sw)i8tV%=4;{K^Bizx zaVp!pC$obgryv&&gU5Bd~Kx6?SnY6fV_5e(`V-ExCcPk|3cy zVUV%9qeP2?gq8;hEf5k~A|$j(Na#i&p(!DuNf`u|Iz}0Sy&H2_n_+XkQ2GB$;W3br z1*bz_*f6TEv-q-xDmI&nYmB9J6_sVyx1BTKwZ)RaCjnhi+U*qSq~k`K*|8V4+~{q)X6{m8qrd-FGwHWw z?JOgt+ji=0m5SVk#(f#2$M$sJKd8xoZQDneRZ#trw{6auFo6u&R{#Etynr@BJ=?W< zwq^CK3+h=D)UzI_XDv|AI-s64Kt0Q#o)xH`edpgkTQiC*rmF^SF2h_6FSGqow< z+#5=0XjbAtd-DRsADhzaYJat31lCWZ z#9iJpcWAh(K<5irQ}MWbdo>3!xjlvS!{+vE#Ix<`h<|KfgqUzGlXOGHwPi~7Ed8fD zoV=baJ=e>;U5(fc!u6%-RB+vZcY5Pfb@zT_62@RM7s5#^4Rh-)1N zKE+fha?F7H4mB#DIrNAr9dwPmJ7}ohjs(0De(1;|J&@M94DsF0&4^Dr*D2U#yBPkt zOAq_If>3#;Ya^n2cM+noy8!Wg_iDt6J?pviI`Hk81i$Rb=3)^OM|x7xC$M)7;?~|G zZZa~HZ~yc|2Ryu-N0QCAg&8t$(*^Cm{TlbKLgjq$J?I9peQK`S=2Gn$Q<>S&#=eEz zV>=Ul`{!e%<^9fd-gYXd2bP((S5=xy%Eoq`&HfIozmo!^R`uBF?tyg!OL)A!pu4Yu z`G+In!LL;CbU?|~^*rL>Je($Pa1B@PXUb<^c*413fiOB4#MOtG+J7h$eG7+*5u1kc zNzb7>Ymt5R&ML%D?vx<<+}(g!b$12gwY%#PBR^eC2B7@YJO%q+Xl9X;KAQsfe;o|o zpDDT8?o==0;$`PDCPDjgKlt&u=U7?;(DPY2_i2Yc_tmiM^H(tZFFs$&l@2!W;pa)H z^uFg*;{AZtt-Uv$`}8xh^&a(kI_C2!Q=Yh@fOTKI!u{?s)yH4V;^G4)KKo)04x4&^ z1>(;8uW{3#AbB`S>eCA$W0UJ z30qcdSy6Ac?y-vRcw*SQ2=VNV6u*ASx_{i-Ws`k*cU}m6DlJ}%ja4(V@kfDfQ(aA^ zm3==+*%3rTIwY@1q9q;n8IdH#PoN(<(!g{Qte~7F0)B3cBAwFjl1MbSMI@16(rN!X ziR>{7^o5a3RuWAHr4J7?(s-C8q4fALNkHN$kvR6SSVCc!Eu0V7_$%i5rbIN{77hi? z5$N3_5lS8)38bOG60^i%ixV`Tnra36>K1#xn96O3TH7H#= zO4nzS;b!aa0~f!fj}QW>BK~thI2j>WuY$JBq`EM=FT1>O9%|KgmE?i zu5DMlu}>>$ze*NI`lJuN#)e%*MK}#=@ukz|IsXpvO=XjnsAO6`#axvvoQDj8rZ`a6}KX4__R9RRU{4xR> zZ@Qs2|FmrCxHdpKa9T#!g%ihc^<*jfj4TwM-SR^9_ornE+!`kJFOa!mA37I#-{H;u z)zGkak_^?m&Wuf~mZEmal+fDYPP*+k&&UQY$@V1K50}UfDFyn3NZ)RjCy*X#ZG}9N zUHaH&ySt)MbG$+x$2DP)-L1)`-WG4Y^BG!cE-}|(7cs5pn6{)8{|j>6q!)F%GS}wgq#086 zS>h&TE)b?ksvIFwo^DtyCFTg>@}-7#k|9Tkkrx?EQZ?1o76l)KmPRhh-Q;bCf0ZKjLZbXh-fSsfFKC2xl4q4ndLdSjNZwiL zuwID6d|LIwUkcb;VhM+nk0RKwusTaDTssqMe{L22B_xc#5&SAxTf5S1swpWq>Nc6G zcUZ+C>G?T9LxvMZW%csOFxHWyUOH0EVC&dH?_b delta 114 zcmbQWj&;^L)`l&N2L$=dEiBE9&GgL7EiDX8jJKZ?Wb|Z37gQBze8i0=I9*hk(G^`R QLz&SEO>FxHWyUOH0Go6h*#H0l diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index 109edca7039022c95e1eacb3f731b944d016c741..616a9c69c5aa5fa0e9acef9dfa829fe388664073 100644 GIT binary patch delta 150 zcmex4o9WAJCYA=)sV_IOv}!QU+1#ZO&&X?GU~Xw*qGxJmU}|o-_`4Source code for gen3.tools.indexing.verify_manifest
MAX_CONCURRENT_REQUESTS (int): maximum number of desired concurrent requests across processes/threads """ +import aiohttp import asyncio import csv from cdislogging import get_logger @@ -455,7 +456,15 @@

Source code for gen3.tools.indexing.verify_manifest

if "https" not in commons_url: ssl = False - return await index.async_get_record(guid, _ssl=ssl) + record = None + + try: + return await index.async_get_record(guid, _ssl=ssl) + + except aiohttp.client_exceptions.ClientResponseError as exc: + logging.warning(f"couldn't get record. error: {exc}") + + return record
diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index edffe0ce4..8ecf6599e 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "initi": [0, 10], "follow": [0, 11], "curl": 0, "data": [0, 1, 3, 5, 7, 8, 10, 11], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "user": [0, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 3, 10, 11], "map": [0, 11], "get": [0, 1, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 3, 5, 6, 8, 10, 11], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": 0, "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "refresh_access_token": 0, "new": [0, 3], "gen3fil": 1, "auth_provid": [1, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": 1, "guid": [1, 3, 5, 6, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "id": [1, 3, 10, 11], "text": [1, 7, 8], "result": [1, 8, 10], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "get_presigned_url": 1, "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": 1, "file_nam": [1, 3, 10, 11], "expires_in": 1, "upload": [1, 3, 8, 13], "list": [1, 3, 4, 5, 7, 8, 10, 11, 13], "author": 1, "scope": 1, "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "represent": [1, 3], "document": [1, 3], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "job": 2, "metadata": [2, 3, 6, 9, 11], "submiss": 2, "tool": 2, "dr": [2, 9], "verifi": [2, 9], "ingest": [2, 9], "modul": [2, 10, 11], "gen3index": 3, "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "tupl": [3, 11, 12], "async": [3, 4, 5, 9, 11, 12], "async_create_record": 3, "hash": [3, 11], "size": [3, 10, 11], "did": 3, "baseid": 3, "acl": [3, 11], "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "creat": [3, 4, 5, 6, 8, 10, 11], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "md5": [3, 11], "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 10, 11], "entri": 3, "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "async_get_record": 3, "async_get_records_from_checksum": 3, "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "async_get_records_on_pag": 3, "async_get_with_param": 3, "param": [3, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "async_query_url": 3, "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "async_update_record": 3, "updat": [3, 5, 10, 11], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "create_blank": 3, "blank": 3, "create_new_vers": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "note": [3, 11, 12], "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "create_record": 3, "delete_record": [3, 8], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": 3, "distribut": 3, "identifi": [3, 9, 11], "boolean": 3, "we": [3, 11], "get_all_record": 3, "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "get_latest_vers": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "get_record": 3, "get_record_doc": 3, "repres": [3, 5, 10], "get_records_on_pag": 3, "get_stat": 3, "info": [3, 11], "get_url": 3, "algorithm": 3, "get_vers": [3, 4, 5], "get_with_param": 3, "file_st": 3, "client": 3, "like": [3, 5, 9, 11, 12], "signpost": 3, "is_healthi": [3, 4, 5], "healthi": [3, 4, 5], "query_url": 3, "update_blank": 3, "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "update_record": 3, "exist": [3, 5, 6, 9, 12], "gen3job": [4, 10], "s": [4, 8, 10, 11], "dispatch": 4, "async_run_job_and_wait": 4, "job_nam": 4, "job_input": 4, "kwarg": [4, 5], "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10], "global": 4, "input": [4, 10, 11], "create_job": 4, "get_output": 4, "job_id": 4, "previous": 4, "complet": [4, 11], "get_statu": 4, "statu": [4, 10], "bool": [4, 5, 8, 10, 11, 12], "list_job": 4, "gen3metadata": 5, "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "async_cr": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "async_get": 5, "async_upd": 5, "batch_creat": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "create_index_key_path": 5, "databas": 5, "delete_index_key_path": 5, "get_index_key_path": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": 5, "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": 5, "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "must": 5, "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "gen3object": 6, "level": 6, "current": [6, 8, 10], "allow": [6, 8, 10, 11, 12], "system": [6, 7, 8, 9], "delete_object": 6, "bucket": 6, "gen3queri": 7, "elasticsearch": 7, "graphql_queri": 7, "query_str": 7, "execut": [7, 8], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "http": [7, 11, 12], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "valid": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "raw_data_download": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "gen3submiss": 8, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "create_program": 8, "program": [8, 11], "create_project": 8, "project": [8, 11], "dcf": 8, "delete_nod": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "delete_program": 8, "empti": 8, "delete_project": 8, "uuid1": 8, "uuid2": 8, "export_nod": 8, "node_typ": 8, "fileformat": 8, "singl": [8, 10], "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "export_record": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "get_dictionary_al": 8, "entir": 8, "schema": 8, "get_dictionary_nod": 8, "get_graphql_schema": 8, "run": 8, "introspect": 8, "get_program": 8, "regist": 8, "get_project_dictionari": 8, "get_project_manifest": 8, "manifest": [8, 9, 10, 11, 12], "get_project": 8, "open_project": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "submit_fil": 8, "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "submit_record": 8, "should": [8, 11], "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": 9, "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "downloadmanag": 10, "datafil": 10, "load": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "i": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "hostnam": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "user_access": 10, "cache_hosts_wts_token": 10, "object_list": 10, "host": 10, "obtain": 10, "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "downloadstatu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "get_fresh_token": 10, "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolve_object": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "start_tim": 10, "end_tim": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "object_id": 10, "object_typ": 10, "drsobjecttyp": 10, "unknown": 10, "file_s": [10, 11], "updated_tim": 10, "created_tim": 10, "access_method": 10, "factori": 10, "children": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "_manag": 10, "self": 10, "pprint": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": 10, "md5sum": 10, "commons_url": [10, 11, 12], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "create_object_list": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "load_manifest": 10, "download_drs_object": 10, "output_dir": 10, "conveni": 10, "download_files_in_drs_manifest": 10, "infil": 10, "list_access_in_drs_manifest": 10, "list_drs_object": 10, "list_files_in_drs_manifest": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "separ": 11, "header": 11, "space": 11, "download_manifest": 11, "current_dir": 11, "indexd_record_page_s": 11, "per": [11, 12], "max_concurrent_request": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "tmp_folder": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "async_download_object_manifest": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "prev_guid": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "threadcontrol": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "delete_all_guid": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "index_object_manifest": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "loop": 11, "ha": 11, "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "altern": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "async_verify_object_manifest": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1654627682": [], "4302042": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "column_to_use_as_guid": 12, "guid_type_for_indexed_file_object": 12, "guid_type_for_non_indexed_file_object": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "async_ingest_metadata_manifest": 12, "metadata_sourc": 12, "672043": [], "get_guid_from_fil": 12, "metadata_typ": 12, "namespac": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "async_query_urls_from_indexd": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "gen3wsstorag": 13, "copi": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "download_url": 13, "ws": 13, "wskei": 13, "ls": 13, "ls_path": 13, "ws_urlstr": 13, "form": 13, "rm": 13, "rm_path": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "upload_url": 13, "1654634067": [], "5923028": [], "8967967": [], "1654789636": [11, 12], "6798043": 11, "9773014": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_update"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "s": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 56}}) \ No newline at end of file +Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "initi": [0, 10], "follow": [0, 11], "curl": 0, "data": [0, 1, 3, 5, 7, 8, 10, 11], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "user": [0, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 3, 10, 11], "map": [0, 11], "get": [0, 1, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 3, 5, 6, 8, 10, 11], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": 0, "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "refresh_access_token": 0, "new": [0, 3], "gen3fil": 1, "auth_provid": [1, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": 1, "guid": [1, 3, 5, 6, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "id": [1, 3, 10, 11], "text": [1, 7, 8], "result": [1, 8, 10], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "get_presigned_url": 1, "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": 1, "file_nam": [1, 3, 10, 11], "expires_in": 1, "upload": [1, 3, 8, 13], "list": [1, 3, 4, 5, 7, 8, 10, 11, 13], "author": 1, "scope": 1, "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "represent": [1, 3], "document": [1, 3], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "job": 2, "metadata": [2, 3, 6, 9, 11], "submiss": 2, "tool": 2, "dr": [2, 9], "verifi": [2, 9], "ingest": [2, 9], "modul": [2, 10, 11], "gen3index": 3, "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "tupl": [3, 11, 12], "async": [3, 4, 5, 9, 11, 12], "async_create_record": 3, "hash": [3, 11], "size": [3, 10, 11], "did": 3, "baseid": 3, "acl": [3, 11], "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "creat": [3, 4, 5, 6, 8, 10, 11], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "md5": [3, 11], "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 10, 11], "entri": 3, "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "async_get_record": 3, "async_get_records_from_checksum": 3, "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "async_get_records_on_pag": 3, "async_get_with_param": 3, "param": [3, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "async_query_url": 3, "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "async_update_record": 3, "updat": [3, 5, 10, 11], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "create_blank": 3, "blank": 3, "create_new_vers": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "note": [3, 11, 12], "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "create_record": 3, "delete_record": [3, 8], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": 3, "distribut": 3, "identifi": [3, 9, 11], "boolean": 3, "we": [3, 11], "get_all_record": 3, "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "get_latest_vers": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "get_record": 3, "get_record_doc": 3, "repres": [3, 5, 10], "get_records_on_pag": 3, "get_stat": 3, "info": [3, 11], "get_url": 3, "algorithm": 3, "get_vers": [3, 4, 5], "get_with_param": 3, "file_st": 3, "client": 3, "like": [3, 5, 9, 11, 12], "signpost": 3, "is_healthi": [3, 4, 5], "healthi": [3, 4, 5], "query_url": 3, "update_blank": 3, "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "update_record": 3, "exist": [3, 5, 6, 9, 12], "gen3job": [4, 10], "s": [4, 8, 10, 11], "dispatch": 4, "async_run_job_and_wait": 4, "job_nam": 4, "job_input": 4, "kwarg": [4, 5], "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10], "global": 4, "input": [4, 10, 11], "create_job": 4, "get_output": 4, "job_id": 4, "previous": 4, "complet": [4, 11], "get_statu": 4, "statu": [4, 10], "bool": [4, 5, 8, 10, 11, 12], "list_job": 4, "gen3metadata": 5, "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "async_cr": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "async_get": 5, "async_upd": 5, "batch_creat": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "create_index_key_path": 5, "databas": 5, "delete_index_key_path": 5, "get_index_key_path": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": 5, "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": 5, "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "must": 5, "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "gen3object": 6, "level": 6, "current": [6, 8, 10], "allow": [6, 8, 10, 11, 12], "system": [6, 7, 8, 9], "delete_object": 6, "bucket": 6, "gen3queri": 7, "elasticsearch": 7, "graphql_queri": 7, "query_str": 7, "execut": [7, 8], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "http": [7, 11, 12], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "valid": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "raw_data_download": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "gen3submiss": 8, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "create_program": 8, "program": [8, 11], "create_project": 8, "project": [8, 11], "dcf": 8, "delete_nod": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "delete_program": 8, "empti": 8, "delete_project": 8, "uuid1": 8, "uuid2": 8, "export_nod": 8, "node_typ": 8, "fileformat": 8, "singl": [8, 10], "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "export_record": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "get_dictionary_al": 8, "entir": 8, "schema": 8, "get_dictionary_nod": 8, "get_graphql_schema": 8, "run": 8, "introspect": 8, "get_program": 8, "regist": 8, "get_project_dictionari": 8, "get_project_manifest": 8, "manifest": [8, 9, 10, 11, 12], "get_project": 8, "open_project": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "submit_fil": 8, "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "submit_record": 8, "should": [8, 11], "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": 9, "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "downloadmanag": 10, "datafil": 10, "load": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "i": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "hostnam": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "user_access": 10, "cache_hosts_wts_token": 10, "object_list": 10, "host": 10, "obtain": 10, "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "downloadstatu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "get_fresh_token": 10, "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolve_object": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "start_tim": 10, "end_tim": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "object_id": 10, "object_typ": 10, "drsobjecttyp": 10, "unknown": 10, "file_s": [10, 11], "updated_tim": 10, "created_tim": 10, "access_method": 10, "factori": 10, "children": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "_manag": 10, "self": 10, "pprint": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": 10, "md5sum": 10, "commons_url": [10, 11, 12], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "create_object_list": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "load_manifest": 10, "download_drs_object": 10, "output_dir": 10, "conveni": 10, "download_files_in_drs_manifest": 10, "infil": 10, "list_access_in_drs_manifest": 10, "list_drs_object": 10, "list_files_in_drs_manifest": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "separ": 11, "header": 11, "space": 11, "download_manifest": 11, "current_dir": 11, "indexd_record_page_s": 11, "per": [11, 12], "max_concurrent_request": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "tmp_folder": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "async_download_object_manifest": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "prev_guid": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "threadcontrol": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "delete_all_guid": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "index_object_manifest": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "loop": 11, "ha": 11, "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "altern": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "async_verify_object_manifest": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1654627682": [], "4302042": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "column_to_use_as_guid": 12, "guid_type_for_indexed_file_object": 12, "guid_type_for_non_indexed_file_object": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "async_ingest_metadata_manifest": 12, "metadata_sourc": 12, "672043": [], "get_guid_from_fil": 12, "metadata_typ": 12, "namespac": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "async_query_urls_from_indexd": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "gen3wsstorag": 13, "copi": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "download_url": 13, "ws": 13, "wskei": 13, "ls": 13, "ls_path": 13, "ws_urlstr": 13, "form": 13, "rm": 13, "rm_path": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "upload_url": 13, "1654634067": [], "5923028": [], "8967967": [], "1654789636": [], "6798043": [], "9773014": [], "1654807944": [11, 12], "3028057": 11, "560571": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_update"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "s": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 56}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index ab9f68a06..16360495c 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -340,7 +340,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1654789636.6798043.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1654807944.3028057.log')[source]

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index 827ab8497..da4286ad3 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -104,7 +104,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1654789636.9773014.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1654807944.560571.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters: