From 0de79cd3deac821d8d6c27b357a2c073b68e4114 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 24 Sep 2021 13:58:24 -0500 Subject: [PATCH 1/6] Add apiendpoint and other params to avoid breaking changes --- gen3/cli/configure.py | 21 ++++++++++++++++++--- gen3/configure.py | 24 ++++++++++++++---------- tests/test_configure.py | 2 -- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/gen3/cli/configure.py b/gen3/cli/configure.py index c8e39a5c8..3f6399fa8 100644 --- a/gen3/cli/configure.py +++ b/gen3/cli/configure.py @@ -6,16 +6,31 @@ @click.command() @click.option("--profile", help="name of the profile to name for this credentials") @click.option("--cred", help="path to the credentials.json") -def configure(profile, cred): +@click.option( + "--apiendpoint", default="", help="[optional] API endpoint of the commons" +) +@click.option( + "--use-shepherd", + default="", + help="[optional] true/false -- to enable/disable the gen3Sdk to upload using the Gen3 Object Management API (if available)", +) +@click.option( + "--min-shepherd-version", + default="", + help="[optional] Specify the minimum version of the Gen3 Object Management API ", +) +def configure(profile, cred, apiendpoint, use_shepherd, min_shepherd_version): """Command to configure multiple profiles with corresponding credentials - ./gen3 configure --profile= --cred= + ./gen3 configure --profile= --cred= --apiendpoint=https://example.com --use-sheperd= --min-shepherd-version= """ logging.info(f"Configuring profile [ {profile} ] with credentials at {cred}") try: - profile_title, new_lines = config_tool.get_profile_from_creds(profile, cred) + profile_title, new_lines = config_tool.get_profile_from_creds( + profile, cred, apiendpoint, use_shepherd, min_shepherd_version + ) lines = config_tool.get_current_config_lines() config_tool.update_config_lines(lines, profile_title, new_lines) except Exception as e: diff --git a/gen3/configure.py b/gen3/configure.py index cd13978a1..10733f473 100644 --- a/gen3/configure.py +++ b/gen3/configure.py @@ -29,18 +29,22 @@ CONFIG_FILE_PATH = expanduser("~/.gen3/config") -def get_profile_from_creds(profile, cred): +def get_profile_from_creds( + profile, cred, api_endpoint="", use_sheperd="", min_sheperd_version="" +): with open(expanduser(cred)) as f: creds_from_json = json.load(f) credentials = OrderedDict() credentials["key_id"] = creds_from_json["key_id"] credentials["api_key"] = creds_from_json["api_key"] - credentials["api_endpoint"] = auth_tool.endpoint_from_token( - credentials["api_key"] + credentials["api_endpoint"] = ( + api_endpoint + if api_endpoint != "" + else auth_tool.endpoint_from_token(credentials["api_key"]) ) credentials["access_key"] = auth_tool.get_access_token_with_key(credentials) - credentials["use_shepherd"] = "" - credentials["min_shepherd_version"] = "" + credentials["use_shepherd"] = use_sheperd + credentials["min_shepherd_version"] = min_sheperd_version profile_line = "[" + profile + "]\n" new_lines = [key + "=" + value + "\n" for key, value in credentials.items()] new_lines.append("\n") # Adds an empty line between two profiles. @@ -65,12 +69,12 @@ def update_config_lines(lines, profile_title, new_lines): if profile_title in lines: profile_line_index = lines.index(profile_title) next_profile_index = len(lines) - for i in range(profile_line_index, len(lines)): + for i in range(profile_line_index + 1, len(lines)): if lines[i][0] == "[": next_profile_index = i break del lines[profile_line_index:next_profile_index] - - with open(CONFIG_FILE_PATH, "a+") as configFile: - configFile.write(profile_title) - configFile.writelines(new_lines) + lines.append(profile_title) + lines += new_lines + with open(CONFIG_FILE_PATH, "w+") as configFile: + configFile.writelines(lines) diff --git a/tests/test_configure.py b/tests/test_configure.py index beb92f317..17f311a0e 100644 --- a/tests/test_configure.py +++ b/tests/test_configure.py @@ -42,8 +42,6 @@ def mock_access_key(_): def test_get_profile_from_creds(monkeypatch): test_file_name = str(uuid.uuid4()) + ".json" try: - profile = "DummyProfile" - creds = {"key_id": "1234", "api_key": "abc"} with open(test_file_name, "w+") as cred_file: json.dump(creds, cred_file) From 7f93139865ef9433f9032ae49e77e090c80473e1 Mon Sep 17 00:00:00 2001 From: nss10 Date: Fri, 24 Sep 2021 19:07:17 +0000 Subject: [PATCH 2/6] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 225657 -> 225657 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 93370 -> 93370 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 34281 -> 34281 bytes docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index ac8da1d581cb2248c63658014bf3840f34f177b1..f31978c56c140b09cac5549de5dc92181c30bbba 100644 GIT binary patch delta 3867 zcma)9X>eQB73MxWlJRbA$ML>MN*ph;oz->}%W_ulu@yo}fLO7W)DLg5BqtQIG)x1I zU%ccfd673+!2tsyI1yl`1==62J8dAuW#~YrErzz!vZR5ww1pJXbMBL5m}zJF=euXW z=bU@*(bE?nM11i<#PqTBBa!yb&Nf>?yVY)KvDht6$tP&-Xt5q_?bzp(7I6z)_dBfJ z4>_e^i^Kk1T*S^jHm4M7bL?qvwb@!bJDgI`0UL@!7J>!uZEdrnE02bnK-59U9!cF} z?e?;aKL|(0ggfroVtU8?_`&e#m&J1dF0%Q3@reKzrT@M-8Q=>0r^N99w|3_*L{EU* zaPd8HG{9AU6(qU?oTKe?aU{SUDEN^$oE>1DAGL_C05|eto;VTUJfFV|l4jjnN_N1R zEk)_|1^cWWD+{O}`mTqZgFN$Zsi=HKH3>h_B?b+OFO%WJTn*%}Tm@~}?Qm$B2Cn9);Pv7q z@bywPWG-)n19=*lSf+x}<(=?uwg&V$huA23W})Q)h{)H#Cq@nQ2!XDMm_ig?4>5-*x*B3JQFJZD%|ub3#qC5FR!WFgN{3cThE__2))AOrm8s?B@yd89yb4|b z&z+||;?=EQE#RIp@Zj1T@Ofc}ug?#{TSZ#fRWb`d+qeeYYl?lnPJj!gX834x3`DPc z7j_nRVBa(0pur4lOSEv)FpFn!#Mq8e(!f+^hOj~{yj+GAS1VCIG4{aQwd5TO$IEBo z)fK5wR+{S639Yg*sWy_%Yx@SEO3thk+cm6&GqOF4S6FsJXb% zg}{Xhs1C-fFxF7Dk-RDA|L1hjln)FmkMxa7a~o6mki!?o(+P|UA4dnipraHQILdT^ zql6bY%6oyM^cOgqTnpSV^fxBKd}F+dznBs4twtu`J^yO^9)>TtX~%^SrBZRy8+VT( zl#P2EEI$;O#~a@AYZV*y{$6Zh%xHXU~_nVn|A?T_h z%FaGSo6iibR}6o^H$U1G7Vg^{n=L*LC799fQ?ZIoAFsoGit2dbR3RIOKc3o*m_(zX@}$1aP%xLH-lM@WRyVeWQ|OQX8F zJ4I`=zh7>M9<9R7c8yjmzQG|dFD>TL-9{loIe0gGI$EV{9t!V|t(IR|FGTxQePi1& zz;DLt5%b0?5qrkXh`$_fKumF0Dvs$;G+rd%E*B#Gj??Z6Wlx=d&pdQ6Oi(_LO=RQ4 zoSxXghT-oM)rjSu2N2yHzxFgBz3rS8@uhPWh+mxBs^Y(Um5@1E4ABoS@?9=2tRZ2H zx$@pkzH7pz^(c*l8&BxbKV{PAe>a(i-`!Ya_M5`R$x0>mF79AEqzB`9l_Gk8i*}zc z#u#VM??t?Qz8$gs$(@L^Pg4G&Px<`2xN-YaB)s&LMTyV@SDs9Vwnvvh{gfK#|2@hErJvSqpSTrctP-Z7fZ3iI=~uXHiJc zHLz4%ZZ`OBuGO*R4Dyo_D5*;NTGzK&+pKo$?rv*yXLk#3AU8C!IAzyFh#E*%!XK_< z85rx?M&=WIUCUAwt9X2#ad+C3+@w_E_$({oJH;;SL_@@8x7hurcYRxh-SKJUOEoMJ zcYD~(3viiPhLXiV4O^m6hHZXoq@E?AZS;1kf2LqN3(De?OG<}%ONJnh%lG!Mq!b!h z5>KWI|AOehInc)CL9|eu;($392a~ZpXJ-kD@GkhQITj<${JV$kE#)&uN`#}QlY?n` z$&u6OO%w{N_a-gkzgzOQX_lJ8rF_>~-+v{2bGOxEw>m|SeESkxu0)8w#1a(hwddJV z=8-!t`vpzUGc5`(US{dIT)m7{dgL3IS>k_-?dZb4V-$NP%gdxRHYz_qhYq9i)j75} zn7#t!`{yvw=uF73SW_Ew%l&0Sn*3g=pi*cppCNeNJO+WT8x1tsZ4lB?ooW=)V7_CK zAaYt8%$4Z|Awy9n%jq|mDj3huE%zCPOq4!n6g0TZ8HH>ma29+v9!pa#46cpQo~T+P zyCa0iAabiM7qU>;RW9I!lE=!099&SCEPuLQ2v$OWtk^`Fgk*oR4JKhR6i!gQ8WZg} z(`6D`3iv-Ao1}yNkHo?J!I+o!3yxiU7)VQD?tFGIO((hQilA47@p8-+A^uJ$|GscY sS?HHFll+D-P{M|1o>c{T>zPa5y(37Yrn%veG+=pDGjm}_&>Knr0is?SDF6Tf delta 3802 zcma)9dr(y870-9U1wj-9MBa~G1OXQ?Di2o{P*mQ+8e?OOtgu?HJXUuXwTX|>i79dg z@vL4KS(XKSC7Pg{hEB&c?MzE&YGS00og~vXG@VXsj5Vfd8)K?H=exU*%;b;$^ZU*@ z-+7;V&)t3aNyx)bLPifIYJx1bt*y-#OLKdhUGf*(w_5DdjOMl`(;j4vHtP@VQXq6D z#^%kowzs!fvRh47W0TQpv`c=wEvOsFb(`C^quEZIY1dvnxh9%#Yi==lzDm3&9OxIG zdTLF{2NgvJ+~4dFN2a*ptBvBBDQ=|dmgt`1u6=o1?3?0#`LZB7r?~4&&En7$ceAEU zbWCv%mWPVNQ`{HJ!o-1eAA2y!s1jXM+?-EG#la~q(*GTh)XVcI*&chkG)Jq=-fn7p zI-A;{HSe__=b3-!ib_q!A(sjB&BA-BhAbiHfwW>1#GElYza71lfEah=>U8hN&puo6g^i8 z_r>;24(_I&WBqVq!Cm+;T@8b2Hn@|ihDA>=ftK`E2+mN$jSLmMn?D~OF9?9-h4rvI zi}X~`zpx!^+n<&K+Khd$FO4*JL&zdEe5O}JSLO>4i5_QV?EuR{HS}d!pe$4E$yt=a zOw?3S@I#>Ki*ZEJlu2G-c6D6iry%PZTvzl!$V{ zJK6-+@A;E<5d(XX2C{R;VPk$9<{l3Ra}AJJpn?D7jzd*RJbbkHBUn(Xfe)U(249zk z7f^yn>@_?+Iw>VODIq#39Xcr)Iw=)8hhcI_vWAz(E90f`DtG}rcb@jJCw1vk0b_>2 zj=XztXL*~q&pY6{P6L|?#^KkimVs+ozPHy=aH-G$pRNvr(B&Uu-w)_pvG2*SM`wWG z91Yypjl;ik$V&p06dB-Ft_FTuG!8$hRJ?qq?||FYWE~DiipSyY#qm&77!8kB72u5K z!>tknJeQ~3SCo!|eMxvaHIu5McjSoOB+Y7WYizMKndmvU*f-nLm3F%zW9e?srIpbP zk_(ri#|!YJP~-VcnVKOH=6SUG6@`GcM-(EXqKm)c>%T-Ydm9B35>ZQrXm#% z9Eyg_HDR9Anq0p%)X7T5$AAYtB9#Ik26EI^JgBjFP;2p^=HfvQ1RhjCr9VCj&bo?K zWKB8$Khr@~z6(s;)3++ktdHaS3twcO&eRt0ay0PiJ4$hyqfDncN_d*1yr(%zf10DE zG|de_cYO>@)<>%N8#C;ARL=x_&i}i9D`O+ju;EgmQmN?nL_DhtVtt-Eiu!8#2$y?sZi|XVJ@8tdN@ zAl~d+%-rB7l_8c(D-n-LRfr!+YgPOOs)8D6k?g8tvsB#N4cm{cMupdp=@I{NtV~hc z2tVphgNE)16xzCr5#Q=whIp^L6fvjAtF|4k^`^mOPf(y;$;J(D_LQK;zk7-lM`ma_ z#x>-40}J-WeCfDOx$b~mh=TFwBjC>QU`4bS9`$6(-3Asg?Q}^mWoPZB;A6cjVDR(| z_A-xsupxMsH#GKI{4o?@LR(IRE55pT8Lksl#+xT{@GJ7y6RQ!kV7ZE~lG7MJHEO%b zx~1J_HQTJ-GWgvYx`P`Kh6aErN~PLcAk$LdK~wR4|{?BaWV`L43k#y-akstV6smS1IQ2!-OLW z9L*_H=9^EiLzM;W z`ztWmrv6IB);|y?rFq=DOD}{dvv+fz80RWi3=<)DAXk1;EX?*9j=D;fn5n*)S!kdeqxay`3If%+MBs@>v|U7ib zqNj?@S18>&A9bdd#h`BJda8dcdp+|@;oD0}gh^wPAoj`6b+DK?T3QlMrUL(MXurMG z`sCxNp%_H~v(NJm=c+eP*p2oiEVA~4$-jSl=;K^8j+FdRZ^1@tDvH>5AhNcrXx7TP+xFIw>Yz&6iyv!fZb>t1cE& zP-rd|a6`$?Vj%+$6voP*uMqqd*LM}2*(E}(FWI^hVIJfRlE11Fia2I25t_0CX!}^C zRM`1a1WfJ;duyj)+r*cFv;Zc~ru);qY`!jN6=9?tc3p^^Y3DyNY*Gr^rHsYCC-fAs ifw9+Aex6$9lDBN|QwPxAut{n#9#)TC+TizI%>MvsIRSA1 diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index f63288d616680ce727cab3b8c9320860ec0a77a3..02cbdee8d2d8b7d8c383ea7e8d8aa01e1653359b 100644 GIT binary patch delta 94 zcmdmWlXcfk)`l&NyH)v&Obrc8jE(gyEDS6xjJBUtWejCO7u3;W)MrN%oStdOXoV)W KeWM{Gr!@e=UKdjU delta 94 zcmdmWlXcfk)`l&NyH)uN49rc9Ee-X|EiKH=Ew`UkWejCO7u3;W)MrN%oStdOXoV)W KeWM{Gr!@f2Ru`iH diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index c51184d08678559b7bcca769192ad6bb1b84fe19..7d772fffd708db0a247d79f45039651ae250c2ea 100644 GIT binary patch delta 92 zcmaFa&GfRHX~RJcej`&u0~2EtJ!2CC6Eg$j%@;HbSug|zbYoc21t(VMm7oGqB diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 72066759e..4c1c1e6b6 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":9,"1630075392":10,"1731167":10,"2280114":[],"255e396f":9,"2889535":[],"333":5,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5964222":[],"641011":[],"6572418":[],"6f90":7,"7461076":[],"7649791":[],"7987797":9,"7d3d8d2083b4":9,"8420":7,"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}}) \ No newline at end of file +Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1632510433":9,"1632510434":10,"1731167":[],"2280114":[],"255e396f":9,"2889535":[],"333":5,"3404603":10,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5964222":[],"641011":[],"6572418":[],"6f90":7,"7461076":[],"7649791":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"8808827":9,"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index 6e2eda5ce..a6b9c8fa2 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -365,7 +365,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1630075391.7987797.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1632510433.8808827.log')[source]

Verify all file object records into a manifest csv

Parameters
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index bb23f4915..348378160 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1630075392.1731167.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1632510434.3404603.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters
From 6b8211f1f447ec7d1308521c1c1e42d0db1a0558 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 24 Sep 2021 14:08:18 -0500 Subject: [PATCH 3/6] Format changes according to 'black' --- gen3/configure.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gen3/configure.py b/gen3/configure.py index 10733f473..58a769de0 100644 --- a/gen3/configure.py +++ b/gen3/configure.py @@ -52,7 +52,7 @@ def get_profile_from_creds( def get_current_config_lines(): - """ Read lines from the config file if exists in ~/.gen3 folder, else create new config file """ + """Read lines from the config file if exists in ~/.gen3 folder, else create new config file""" try: with open(CONFIG_FILE_PATH) as configFile: logging.info(f"Reading existing config file at {CONFIG_FILE_PATH}") @@ -64,7 +64,7 @@ def get_current_config_lines(): def update_config_lines(lines, profile_title, new_lines): - """ Update config file contents with the new profile values """ + """Update config file contents with the new profile values""" if profile_title in lines: profile_line_index = lines.index(profile_title) @@ -76,5 +76,6 @@ def update_config_lines(lines, profile_title, new_lines): del lines[profile_line_index:next_profile_index] lines.append(profile_title) lines += new_lines + with open(CONFIG_FILE_PATH, "w+") as configFile: configFile.writelines(lines) From a92b1ec02a349aa0c45f023f082fad6c7f3e6f97 Mon Sep 17 00:00:00 2001 From: nss10 Date: Fri, 24 Sep 2021 19:10:37 +0000 Subject: [PATCH 4/6] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 225657 -> 225657 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 93370 -> 93370 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 34281 -> 34281 bytes docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index f31978c56c140b09cac5549de5dc92181c30bbba..2270ecb9976b1fe62e7123578cb26a175b0dcea8 100644 GIT binary patch delta 4115 zcma)9eNnC*!J|t{_}gE z@Avt*_nAB2e;WGzr=e3%C$0}{Guur~CcDWgEZ*lZcN}mEvi&w2a(0WQ6;JuzmR2(! zCWrk|l(w`tnIDI~#7J$B&9SGg#b#@_Ou|rfz9(b_!w4f6aV&?{|r}xWmKT zqD2oFJ~BRcKJdzVe#p-So%@U*@pHlT@9|@PE-mOXKkDbS{o8n#pUbbC;`{wv(L0~; z9zXZ!?P`9|&pq~*ns=xA+2<^s{J5X%`Q`(Dz|T2vya$51AeScF>r54vYPD;c&F%NE zp)Jr_4>(V-%zvVyO_Ee&a915iMj=)c3%^NGfmah5!Wt{7Jz&)&kwNIwXad~Q1#n%X zCmArSiGik+We}E92zN9~7CY=Mtu}C^l#+3H0|R}Pk^w(xB4sD|0Z3SBBg1fV<(Fg# z4y_VlONJUU?_US4scjIQs)lRn3V1tjIow?t04vgJ;ozz?7)-N3Xr>yp=?7u|YHG6| zg45OTSG^j#Gai8pNQbO`493u=f3*#Mk8!dxZSXn9zFDP#FEXucemo~zw1{=`(uSHZ_?KZZR; z8u;k`D>|B_5MXY%ceI#o&fU(`N}i7QsI!t)LQk4KJ!$Urq?yyRAKq9OQ%rL?>NE=> zE$xl1jwUl5I19w$NbO+tv(j1BtYB6vD|5h`zCMd1Bd{y?M|f?m3JUXUe1m!ncIHMv zctIrW+Ass(X4!FAF)*2Lg!i*Gpe~q!SIcrBQ8x)6R7iHWbu;kh<{TKw$-@{4U??=g z7h5IWuL@`2^MZC9Y$sgRY2auP`ftpEJDWG)fS1Gj1|v8NHBh=~s(@zpKYgK-pmozZ z{Cvf1R^yPqe!q9JSV>@C6%9TCb=4a0TU*owi3snVt80HScI4&2mt_-> zyFuffE>9pxL_m6JG#q|f2^kMWcvC9N36vY8(r#Amhn1T&@I>{EN;AdEXu=B|&3l2P=`V1!iUp27jSHN6nUAGLCs>J8IS4mul?wKH z4tnp@5{?Xef84%@;49NmHyJ1u&yRVd9?}JoQE!du*Bo(qLz+HRkbdu<_$ER;-fgWx zLAb4qog0bU+t)b~Ovb(0Pp8VsfOqH7bFyMOlrA=tE>>(l}KYk|lQ!|v`Z%)F=DfcS3r zS~3PQVGCl3P>gt7C`bHQ*rs4_VL4O^nc_$d2~jZjF=#%%6%AfGu1Eaa@hy_s4tSzx z6*Tljq0rG&gm}3p2k}pII|p>01SJy+MIaDVs5PrPqKK|LiT40$HK;II|E{ zMiM-K=Tj$ilDZSJAqHk1j)Kom1WTd=aHlsz>@kvng`n&DXm<8KdiiwUM#=D(?D8WG z!6Cl8X?vc74VY2=$wICCXf=-Y_`{19CoY2 z?yH082B;3MK?FL?feumSEI|B)Gau3E%p)#%$616JcB&+Rt+tB+&^M-p{io7U>N&MZ z!KBkTvrO2rH`F&X7o?xwf(F*pdc>*Im5BEktrscXBkB;Zish2~b(nR{uOl9)>@N*q3-@*am(wnj#DF>q&rjQ?_7-zIycSdS$8VFsZ;J^;)Z{?%Mgn^k06dP zyz8k!x_Z=%czJXa;`gK56l{f+z>4ua2!C?1?*U`Naw?24mh9Z(dsLXT3Z+qSYfOv& z@#8-K2dQbugSF+xc~f|4yhMtbDwjs7XzXNgR zd76LFIiJ6k8MmFI!i(ojQUv^2dp;3bpIHu769M>j^w5M+!M+*}GV9+?R7mU*#%8=w zgEj|Wcvuo1Wx{V>pr2KR=Nk}5&gYN;`0#ul;?fHn5i2ej5PL7=D%dB;AQ&do#G4k` z5@`+tAfL=d1Kp$!@xWvO;;WN-#BV3_aRym0sxaM3hOG>bA&z?|UX0FH((NXMK7iko z2KqVK=;&y5vL7Hw{j-YgK0yWfbqO4P=qUe^9e~@e?Dh&W8~`>6`MUnCQio~LA_;y0(j-!toU5(<1j!Q(fO3x+0T>D~slgA+QJfA)C=|=`b~$wT%}11IZ7FhpaLc84w4p zvK4r|Zk5HzskwOFB2&gQ$8h>4EI4+wp&PXbx5|{Y%p&Ti77lBx#e{Dw(=GowT|HYi VRPPW1n=!nW{?)fT9gaQP{{Tl%btnJ; delta 4166 zcma)9dsLI>8Tb7X1A^QthylVaLAeI7DhQE)h|2vEYn|3gfXMh7NF)hpJGGZHTjZ0A zPxO;(f?}nWS~RuvY_?~Atn75QR@T#u50@Z0l%s3QL&UvN?qyo1>|{&1P$}bT|cBw+%&si^1gk+sr0(WzkUMk2>IJ5(2tR zogR|-JMQQhw`9q-k`D~e9b6v%8h_T$g*Jc8pYwBJ+NbzQKbO-t#gF^Byj^ec6MnAr z(k*_}&y{~8<6VBvVg8CA@pIkTKjw#1{jBq|7JkUjja<#--F|N3tJgqK7v|Aq`<cZs<6dC?403)lpNUr z?}qr*HZle$SAPz+v@~eTjfU6P?1EpVsv+yabzn|yhmb5a+(=iz+xe^D-s%8IN^8Je zkE}_B(KHK`WvD@$ei%N*^+tL(M4{)O^lIqMcofc|=kmPZ6n`ADcVBbN-5et z{s^UpDJ4*fc8wpV6zv#4g_J{(+bzvD=hDN@RDR6!8*LVW_i{CmoihhX1$I0L9_|-t z;2*hjuq$5$AFlln@-}MVg9oo=()5G?Q-{5?&17@#b*9$vwC~59HLM1@(!A+94ma0D z8)$Zvqq)(QW^v4E5|+1hG@BhQCVCTCAQneD2v#{On$^onWmU2Q-JbOIIUHsb0gvU~ zhO3nuz_lUY+os3h{ptt^FI)k8HqK$Ij_TXNy+(oUiiYU|0|e)!fTr+GIHK>s)+NC~ zodKGPHE>5a2UR7$NbyioWPsJ1HSk)|9K60(1w{q**xD}GohL;YB#{$EgP}T ztKhv71MIBRK*i>nLYn3OoDuB@t)6z{yQ^m%8-n!p-Ja=EC4sA(Xz(d;7HK>`D^n9B zB0P7uJ}VKh{g_0gSM(D2v~UA_UO5HVbsEoXRXjl=0@5p@;ONsz$k-O)NvWwK@a7h& zyj!*1@Yztcl2t!&CKiI3-LWs+=-5eBzgUishi41!NyIjF!$dmhYs+_nzyN_LvmC`=+ zur~Ctf_qpKdf4mlVK2Xjz4{*Z;(OTX^{}(+VVU+|FSZ=tgA>htd>!KZ$4eA6CZ7x0 zr=wy0PIUnPdLZ&+u(vk{bMNadLA=?!mP~?7C_^k4N)b;8Rfr!7+ZF5~RzZ!BCA#X# zas_jrgw_*V(csbvJ>s8Flu2g0;PJjS(AXD=LPuXQ;;Vfd5O4QwM$GB=nzh2U!8NeZ z9~9`6vYCWe`b*H_@BKwmAS;+pFbh#-Ai=&pe=wkv)LoDZ(J=Q&Bz!s$EQt=m-Tn-* z&p-kegRUE-+1Ur_=CgyFB*P!F&7W)xUhdtS9?x^I5i{C%a)o5m%j$5Qq;oeEf}j z$61URda68tO}EPdFnCT0-KWw}I&o^Vf=OqwXIZdoe~7nd6Oewo3=OQO^@ua4YY^`< zS}#(%Pt+k^7po-qcVT`g8iv}E{qB3ul%m1JnIgo6Gc-uz&{{GIh9Ny74CN!fJ+uWe zVpvC9P%~T+z-I2905Fc`z`Mgy3da8hZVAy4=8h3(bX>6Sz+IpnDVH4nNgeXw*ByZ& z(kU~yyG$Bp?X>WwX5X2)Abhj}H`_B>Df!9+VL@2QqPz55h}3)+d@))fZ5{+4kF6K4 zZQ{axs@}0W4DiudEn?<)Ibzqi0rBVK^@wq+3Pa>sMU`^8+S&vC}JS=uwjw`V3g z=-f1)XWgmzQfJ(yWElSDu0$-Jcofma@ZLl{(sgG|h_9aAjQI80?Fu%;${}eoAHttl z>b+o0SVe^qhVtEI-mAi-wJ42*+vl|CA2;dse~6kcf2g6#;4_6wljTzEJnk1@`i-;a3zd^=+M^Scn|o~QW-UGVx_nQ{9CD!hEbC`IUk>(3{E`I%Kv zI~72N;NdBQf_*X^X4XHQs+QO%8JqD!J=z?8;SouAj0ykxLInzoUTj2ky|{t6;g>Jw zBZghvgjjvC1hM~Oo`QXUj6%tDnt011TPF3v4f5$cG|)}!5D!fkB3_-=BYrhqfIY~0 zNrmawFf=nfjyU9*dMTmkOH^YWiI-fW z#DCS2Fr??}i3*oH^*)=MH6%8X`U%lAsdD;PLEB<7o9w22ou+0>XA5p1);EwyX;&qL z^~Xx#kJOMvjCHYrcm?;WNt|RAg^xXQnN7+~P*ILe#*J*J6^lF3kz}(Q?LN~7zOBO3 z@oL1&RYZxqJz-!43>iqGltq6PStU`rIv+JsOJdMAdkh3 z-CZOmj*dTpB~yX_f3*KS9>>K2w2+)4fjC#nsj)b3C()Ad0r;|c1xA|vcNf`Tz$TcW zgeOnM%IPc=Lub&NQYft4AG4JG7K(K0%RY}abXl*$tbiIb2#h7v0^SA7Zk>dUu@#!Qs~bmo6r(2)|YI33AYk* z+%#TQ3EgqlTEewt2hcfY6H?&74QXDN_f~3FHveV0=bBJH1mjtBj??Ltq#B zVVl8j=`^;Qw9OX$+sC`bBUYJ;xW!ScED4u4tg>i1H5cEt$dqx+F`T}HMaRx|bfXsG sR++MaSww!@!eKRAjQCoaZq;|`_OW3@^$sDh6~k-ke|)>s;b_wS2fVX}W&i*H diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index 02cbdee8d2d8b7d8c383ea7e8d8aa01e1653359b..8dfe5d63758110b4259eeca655fad5b6a0a97fcb 100644 GIT binary patch delta 82 zcmdmWlXcfk)`l&N`&GHkjE(io4GfG73{AJ6Qe})_K@ri_Vl-k$7MY%9$Y_f!uziyu HBZoBry$BSu delta 82 zcmdmWlXcfk)`l&N`&GG3jE(gyEDS6xjLf&6Qe})_K@ri_Vl-k$7MY%9$Y_f!uziyu HBZoBr%5)VI diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index 7d772fffd708db0a247d79f45039651ae250c2ea..308e114751ddcbc218fac2c67a8a9a16e744e1c5 100644 GIT binary patch delta 76 zcmaFa&GfRHX~R(sZZl&OJwp={GeaY@%@;IGSx^K7bQ4&S1twP;xgqm6pD~gR1^`FE B6k-4X delta 76 zcmaFa&GfRHX~R(sZWCh@J!2CC6Eg$j%@;IGSx^K7bQ4&S1twP;xgqm6pD~gR1^`5c B6iNU9 diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 4c1c1e6b6..ffb0a257d 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1632510433":9,"1632510434":10,"1731167":[],"2280114":[],"255e396f":9,"2889535":[],"333":5,"3404603":10,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5964222":[],"641011":[],"6572418":[],"6f90":7,"7461076":[],"7649791":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"8808827":9,"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}}) \ No newline at end of file +Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"11e9":9,"1394515":[],"1446126":10,"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1632510433":[],"1632510434":[],"1632510633":9,"1632510634":10,"1731167":[],"2280114":[],"255e396f":9,"2889535":[],"333":5,"3404603":[],"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5964222":[],"641011":[],"6572418":[],"6f90":7,"7002015":9,"7461076":[],"7649791":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"8808827":[],"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index a6b9c8fa2..ebcb5ae0a 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -365,7 +365,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1632510433.8808827.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1632510633.7002015.log')[source]

Verify all file object records into a manifest csv

Parameters
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index 348378160..a8bd57ab7 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1632510434.3404603.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1632510634.1446126.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters
From 58f2661fbeedbd07e66f8f4e60cba63d4ae33d15 Mon Sep 17 00:00:00 2001 From: Sai Shanmukha Date: Fri, 24 Sep 2021 15:23:17 -0500 Subject: [PATCH 5/6] Add more critical test logic --- tests/test_configure.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_configure.py b/tests/test_configure.py index 17f311a0e..92e29367b 100644 --- a/tests/test_configure.py +++ b/tests/test_configure.py @@ -61,8 +61,12 @@ def test_get_profile_from_creds(monkeypatch): def test_update_config_lines(test_lines, monkeypatch): file_name = str(uuid.uuid4()) monkeypatch.setattr(config_tool, "CONFIG_FILE_PATH", file_name) + with open(file_name, "w") as f: + f.writelines(test_lines) try: - config_tool.update_config_lines(test_lines, expected_profile_line, new_lines) + config_tool.update_config_lines( + config_tool.get_current_config_lines(), expected_profile_line, new_lines + ) with (open(file_name, "r")) as f: assert f.readlines() == [expected_profile_line] + new_lines finally: From f90ec3bff24c3564fd8975f0ad61133fd5d0bafa Mon Sep 17 00:00:00 2001 From: nss10 Date: Fri, 24 Sep 2021 20:26:14 +0000 Subject: [PATCH 6/6] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 225657 -> 225657 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 93370 -> 93366 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 34281 -> 34281 bytes docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index 2270ecb9976b1fe62e7123578cb26a175b0dcea8..07ffa215476346f6407a1bae87f021bd4d3d1d4c 100644 GIT binary patch delta 4093 zcmZ`+YgAO%70$Uc3W zy^cJFVK5TKM4cqGt4UUWG;}p4CbUggnw3_XwySwrlO}1K+G_VXcOK1J$)E4+ zz0cnJ?8mwL-uummk>7k6dHJc-!y)_Z#=e6NA*8#v(|Euk$a!iU`LO_ZJ;%sTWCz%f*PiCb1Kh3fEPgt`ebxRE2&&Q|8tkwm zTUe^mYAgpGBP{Z7ZP9{ATQ0bxN&;n87^JIXi5q^FnVdSpf}{q~eqXE` zl7$B>>Z&CZzSCC0Yw85hXU4&+>Rgh`xY*2W=*nzO4w|!GNai#RbPXS5=HXWC~7ZeG2FXJi7RP;+#39~npL6htC-qfrf?01)Yxn5>o!^)yB*mLJWb)Kqk$zrR~j^3X~=Y?0n?R+ zOILa&={g2cmFa3$0G2yTmgUJ(WZ6+&R4k}`!s}l56M|vHz@tSs;ng)tC@X39wd7IQ zStNzLYr{P3YwY#4B>`qj^w8R*236@iJgMu&7Nx@htsdIT)o@EY4P_#kqJzD+@2j|yD!OgWw$XjRg&Q>N8Z0KR#I=H=g1|ka7-d|Ly z2of>g+gqNI2-tR1B64a534C3!9zLy|fnOJ@y>oRb1c?~PsfmZfPbEU`));SQLmh#a zYNaOJXxtC))=Swu*7QLGtxqjs#l?k|zm{VqMvfK>7g{b{Xu)uyCBub2>bTH5Q7dN` z0_~<@y18;Qb%LE^jgzk3_m>`3*&Q-6ht!JQvT?|5NoL~N2=TBMovi z;2=li4sta7AV;$Za`Z9>xrrzrOO;0WarjM3x`NXDS?}!@!jUQOf48?2G6`+1vtd%w zyvrN+pf(&HIvVAD#Bhm&t)?h%V&^Xv#OeJL-${tayUnyMoJ@F6^t&R+xVP}BY&n_s z?mT*4R!LhizzR0Nk{n=t7+`G}U|kqsO&DN37+@6}VD%bc4H&@sZ$7pQr_AKBwTSN? zt5C4(UkLf96JTAdDwKaU4C9}I-Gc=f_t0Pk;yZ(Dhzn#w6=JneiFjP7LwsM@reKY$ zg9af_bTyNE70lcPUB|be!t=*S-^ChX^2WaiSVEK#E4ef?uSB1fcb~x;Ny`9 zNpuiy59f+QdJ-CRy7mN(&US(xK6hfHr1(R2_>s1VdwplqcK!%9U_^UP#!5PaEDzU7 zn#YSL3&<$^<>VH`)ljNn!4VJZ|(6pDAtxhTKk zW!EMt+)V$OtIHVtk{B*VCdlxgX z9@2pBoI(;k!bE${6{C+c=k_ArJ=cTS^Xv}9`DZH;!_WKdElj!RJQZF!Z;(9n!?kBq z!SwWUXq*YfFNX(b^a}P~Kg6_uK2tBTM;M#?Tr=t%dhTIKc$5jhe69wCWf$5IT^H7q zY547hV#K8vHzL+wtUw&TSfpSd31d((yGp!amMxJwFb(qAB2>`MY7r04mLk41t3&*H zmgdlKNr`A;cnopeJ98<%B#{m@A#y8zA6C%M!w!3&$-&+dN?KDm{0d3@ zs4x@CYAlBGy5G!rE*x0?Mr?=+C4G-@X#(4ea6*S@CHXfoQ2d-{wW=DtomKx}Ryanh-Y zuyi;{a{pulNkd;3TZm6^r=BEBTJiXN$1Sl+u?b4bv0=D@ofI29iB2J_&0zDZ-uLY- ztd4I-yi!LJ@wCVFECHvUq)AZ>*OBECrET(4qm5(*>PBy;`OoEVC$bDS`h-+iG^BC- zsJOGAtVpI4Phi2+;LnZb|0m+8ID#6IQ5+D*GC5Ti7i=U!65a=2bi|^kx&QQ&y(MgD z35jszRFa&IKQZbuT2l&zwR=~Du-Bj1beSlVnUtMc@B4ewbo3bwHlu@gi+8V(Rg#D3 zDx%-qtj4yX}MI~ zR_eGESnLhqct)$`Y%^8MrHM@~Bt%TtaS3ATO`?!v8$IHnj!Q?YOFB-4%Yu%}mV9Tx z7Zb5`EJnaP|#+bWw& zOrn!RV0ZdKtKMerGjtg>9cKK=;-|$YEixsU7RM~IbX;Du$P(mKU3}XtOH5{l(ez;m z8us;|8P$lk$P!zaM%=eG>=u*RfDbI&TmD^pd)U08`*tC$3*BqzU$@O+x3_El2X&ok A-~a#s delta 4065 zcmZ`+dr*^C7VrHM1A@E?h&;k8K>;O#FAyRD5ta8NmbzLC0V4G)kVq1&ZGB8TwM9Nb zJ<-n)APHclN?SCo?CjY7wM%E#)|T$J)9rLC+uhD?AFl0oySrU?bonqG zuvs5N>AvoE;}dovoM~4xZeMqYT?nz*TD$gHEc;B|Xv~{h(KUEM?XwSTcGA`=1oaww zyd?Ec+>uf4fd{sgex!flV(_)~{78Tcx$rsf3UHy#@AKmUE-U0JKNjFLL)&>rfGcd8 z=7$1Y$vdC&o&fjQ?RtJVz&-w!ig#xQ*cVK_{6v5o`0fMV8Q|=nybpq^D4!-fXwMWP zG@8{N#_qMNX=OC#gZ7gw^ZQh^c9Low?y3^W7$m3@v@}s_7VVGVFNNp@7(cuAGR&t2QD2F%Ht6)b)Q_4wpqMzIkW;1eP?{}BBazgTQ z3mJve%fBKcaCn6XzspoX&f0a*nb`&9D^+kKTLEtuEQ7ntgCH%d5l?q`MJ5bqncz~k z3N+b=AQnAtWFLS~a z4frHV(JR1bQ;N2qFQ61{J-?Mww3_@5O3}jct(5YBM-!b*%ft(mr#=G3*dEW2a&T9zAKX=Ze#dC`?#Lb}rHM^`5-ElW?84#m#H zj>1ksO_?3<^k%Qm<#4Ykcs&0W9M*OF>i#&imZ)L(##yZU5nUH%s=(UE!Bn9hLUS`f zU4*CD)Qx>egD13luotW0wsscYSfhmE!Ul|z45h_-*p)Ax?B~U^@I_HK_Nf=HYt`Ok zCC_u%toe0mh#Bjj_D+?>6Zk%N1Kim%3BSoxd*3Qo5hS9#ceXw+5wQK3L}XX?6WCw3 z0luo5gr<76ccwa-AQ1)Gm9cQ-nRr;aEy|lwTTLLXup7I70PYp4;mP_>YUvHABCP$m z(B4;ZtVYPuD&a!wgbS?{F0@v-(5m7>n_eYn9Y;SWOf-~jp+WZ0KI1CgD|V57y5?wS zWC5ucyG9eRvN4GbDtt*StH5*jaWv$Dj>cZ#Xu=B|&3l2P=`V2f6bl@^tqYtx+{aQQ zeAe*W#xwD_hgf~<_zs*e?`j~%d&^`MWnppSK+k2Ro= z)xVEbtB=*Hj}^ZUYrp0AZX6)){ZyM7d

mb8%xf=02?u*=F`!VO+Pz_>oh&bYo~MZt$%)cE8;3BQn34b4im_T>o8iI zO*X69X7!ze>6}ytHy{cf=0S_7v=<@%%wC9Sw-*oxykjpx3_nv5#HP>XAUHK14+qX< zq11C`vw}%yuxB~2b8nchXAa0dTaE_ivpU4-v$cr#7_Ad2Js@fkuZz`^`@1mbhy%yI z^nm-Gb7g4YIaiE0e~t!8b*v#npm*pH!BK$twqpxo)R2~VpmwM-h|R>?L0}lpg?EQy z6^#EI+!W#Oz+EbW=$-*Xi@Xx%iQ=iQw+g8%NWLM-t-is)i^ z*VBM>{g@H)>eyz)AI7#T*tn{Iw21L3CZ*K%DU5#R9~LOPdgDE|nq=4oyrn#nq-Tm z4md$Rm5&D6DJ|l`sUpPJr*w$lPth~fzN|#-WOy8L!aMnLY+*c|XhPUF{0uCmUx975 zo=!XanSsndE7`mglu+1|%;7gi{9p2daKcsHP9r1UuG`NV$hi1zJyElVE+mNiYe<5^ z&c>{0sv#@nbl!<|wL~RRE%hW>a)}lHT}L92o@*dVTy8h`ZQiRTiK*02h@(kW(64n( zyRp+~H8%Gc+e|&}xPjQvNMfX2;~`=&Q3`*!mZV~=OO3=QxLZS#B&%3_;4zCWQf`8h za%>uIWIIJK>_o?q#cHtnO&`2ph1KzC#LLwr9(Q|6&ko?wlT;~-!D_NhqO|pX%2`Je z&^B@hJ^#$=9YmJSW}lD(^M+K89}{==l7u8W@&uMlCI0=;{QpQC6HlUr^G^()MsG@?uxf9@BKG?y)=v{<5|grB zYkdEtG;KXbgVku~9pb&qBuk1Ad6~pX)SIu8<-{R&U-1iCUL|T2ymE!4;Bw;%p3))S zxKr-@i8toRQaSyy5ckYsprM)I_eo19@rVP(T(Wqx zkW)ysnoS_QZXPX1uNxhQqD#xApjxTpl3~7k5yvxHEoaIUEte`O6UCHUL?Oo>dc=Mm zmxj`pb({*9IUScN1x|-=N2BRX42Abb=$@!rCc477a2a*0DdEylXfEM!zKJ6xTsAH! zOccM_#L1=5Ur9FMrCg#v*@jYXDdf6oyy{ZAs;(&JNr(t3F+vUK9l$kH@K0wXD%sZ-a0ZNHaE#s zL3H-m1Qi&bRn1H_$!6jc>DUn1g?`wgx0-qk9Y#%?3IF%;PVul=rX)^r*epxK5_;#wkpU7q-W1 vFdjmc(bZyPVrR6QK0%dHPZCLsUQT|x`jiaz*eMxeJrHRW6E_($a##ZZY^WnV delta 129 zcmdmXlXcfk)`lsJm5i2Ks~A(28Q*Sypv-uWk;lNy*jUfpz`)4B&~*DrRmLeyj2E}Z wX)qo_lhW2=WMXHupFTmAQBMk4lU`1Ky84t1_Sh*IVm%OfR72MpGICf00O*t>-~a#s diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index 308e114751ddcbc218fac2c67a8a9a16e744e1c5..c899f1d8d113698eadc39c2b5b7c36c98d5d9f0f 100644 GIT binary patch delta 80 zcmaFa&GfRHX~Pi>9#cbeLp?K7Q!{f5)6Ew&OjuAw1a#wBQA8$J8@ZwgY(8To8w>!P Ch!t)C delta 80 zcmaFa&GfRHX~Pi>9s@ID6Fox{6Ej02v&|PYOjuAw1a#wBQA8$J8@ZwgY(8To8w>!E C%@ra5 diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index ffb0a257d..cee454efe 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"11e9":9,"1394515":[],"1446126":10,"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1632510433":[],"1632510434":[],"1632510633":9,"1632510634":10,"1731167":[],"2280114":[],"255e396f":9,"2889535":[],"333":5,"3404603":[],"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5964222":[],"641011":[],"6572418":[],"6f90":7,"7002015":9,"7461076":[],"7649791":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"8808827":[],"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}}) \ No newline at end of file +Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"11e9":9,"1394515":[],"1446126":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1632510433":[],"1632510434":[],"1632510633":[],"1632510634":[],"1632515171":[9,10],"1731167":[],"2280114":[],"237243":9,"255e396f":9,"2889535":[],"333":5,"3404603":[],"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5964222":[],"641011":[],"6556785":10,"6572418":[],"6f90":7,"7002015":[],"7461076":[],"7649791":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"8808827":[],"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index ebcb5ae0a..154d211b0 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -365,7 +365,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1632510633.7002015.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1632515171.237243.log')[source]

Verify all file object records into a manifest csv

Parameters
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index a8bd57ab7..76ed17e56 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1632510634.1446126.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1632515171.6556785.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters