diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index ac8da1d58..07ffa2154 100644 Binary files a/docs/_build/doctrees/environment.pickle and b/docs/_build/doctrees/environment.pickle differ diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index f63288d61..fe17685bc 100644 Binary files a/docs/_build/doctrees/tools/indexing.doctree and b/docs/_build/doctrees/tools/indexing.doctree differ diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index c51184d08..c899f1d8d 100644 Binary files a/docs/_build/doctrees/tools/metadata.doctree and b/docs/_build/doctrees/tools/metadata.doctree differ diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 72066759e..cee454efe 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"11e9":9,"1394515":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":9,"1630075392":10,"1731167":10,"2280114":[],"255e396f":9,"2889535":[],"333":5,"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5964222":[],"641011":[],"6572418":[],"6f90":7,"7461076":[],"7649791":[],"7987797":9,"7d3d8d2083b4":9,"8420":7,"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}}) \ No newline at end of file +Search.setIndex({docnames:["auth","file","index","indexing","jobs","metadata","query","submission","tools","tools/indexing","tools/metadata","wss"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","query.rst","submission.rst","tools.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],objects:{"gen3.auth":{Gen3Auth:[0,0,1,""]},"gen3.auth.Gen3Auth":{curl:[0,1,1,""],get_access_token:[0,1,1,""],refresh_access_token:[0,1,1,""]},"gen3.file":{Gen3File:[1,0,1,""]},"gen3.file.Gen3File":{get_presigned_url:[1,1,1,""]},"gen3.index":{Gen3Index:[3,0,1,""]},"gen3.index.Gen3Index":{async_create_record:[3,1,1,""],async_get_record:[3,1,1,""],async_get_records_on_page:[3,1,1,""],async_get_with_params:[3,1,1,""],async_query_urls:[3,1,1,""],async_update_record:[3,1,1,""],create_blank:[3,1,1,""],create_new_version:[3,1,1,""],create_record:[3,1,1,""],delete_record:[3,1,1,""],get:[3,1,1,""],get_all_records:[3,1,1,""],get_latest_version:[3,1,1,""],get_record:[3,1,1,""],get_record_doc:[3,1,1,""],get_records:[3,1,1,""],get_records_on_page:[3,1,1,""],get_stats:[3,1,1,""],get_urls:[3,1,1,""],get_version:[3,1,1,""],get_versions:[3,1,1,""],get_with_params:[3,1,1,""],is_healthy:[3,1,1,""],query_urls:[3,1,1,""],update_blank:[3,1,1,""],update_record:[3,1,1,""]},"gen3.jobs":{Gen3Jobs:[4,0,1,""]},"gen3.jobs.Gen3Jobs":{async_run_job_and_wait:[4,1,1,""],create_job:[4,1,1,""],get_output:[4,1,1,""],get_status:[4,1,1,""],get_version:[4,1,1,""],is_healthy:[4,1,1,""],list_jobs:[4,1,1,""]},"gen3.metadata":{Gen3Metadata:[5,0,1,""]},"gen3.metadata.Gen3Metadata":{"delete":[5,1,1,""],async_create:[5,1,1,""],async_get:[5,1,1,""],async_update:[5,1,1,""],auth_provider:[5,2,1,""],batch_create:[5,1,1,""],create:[5,1,1,""],create_index_key_path:[5,1,1,""],delete_index_key_path:[5,1,1,""],endpoint:[5,2,1,""],get:[5,1,1,""],get_index_key_paths:[5,1,1,""],get_version:[5,1,1,""],is_healthy:[5,1,1,""],query:[5,1,1,""],update:[5,1,1,""]},"gen3.query":{Gen3Query:[6,0,1,""]},"gen3.query.Gen3Query":{graphql_query:[6,1,1,""],query:[6,1,1,""],raw_data_download:[6,1,1,""]},"gen3.submission":{Gen3Submission:[7,0,1,""]},"gen3.submission.Gen3Submission":{create_program:[7,1,1,""],create_project:[7,1,1,""],delete_node:[7,1,1,""],delete_nodes:[7,1,1,""],delete_program:[7,1,1,""],delete_project:[7,1,1,""],delete_record:[7,1,1,""],delete_records:[7,1,1,""],export_node:[7,1,1,""],export_record:[7,1,1,""],get_dictionary_all:[7,1,1,""],get_dictionary_node:[7,1,1,""],get_graphql_schema:[7,1,1,""],get_programs:[7,1,1,""],get_project_dictionary:[7,1,1,""],get_project_manifest:[7,1,1,""],get_projects:[7,1,1,""],open_project:[7,1,1,""],query:[7,1,1,""],submit_file:[7,1,1,""],submit_record:[7,1,1,""]},"gen3.tools.indexing":{download_manifest:[9,3,0,"-"],index_manifest:[9,3,0,"-"],verify_manifest:[9,3,0,"-"]},"gen3.tools.indexing.download_manifest":{CURRENT_DIR:[9,2,1,""],INDEXD_RECORD_PAGE_SIZE:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],TMP_FOLDER:[9,2,1,""],async_download_object_manifest:[9,4,1,""]},"gen3.tools.indexing.index_manifest":{ACLS:[9,2,1,""],AUTHZ:[9,2,1,""],CURRENT_DIR:[9,2,1,""],GUID:[9,2,1,""],MD5:[9,2,1,""],PREV_GUID:[9,2,1,""],SIZE:[9,2,1,""],ThreadControl:[9,0,1,""],URLS:[9,2,1,""],get_and_verify_fileinfos_from_manifest:[9,4,1,""],get_and_verify_fileinfos_from_tsv_manifest:[9,4,1,""],index_object_manifest:[9,4,1,""]},"gen3.tools.indexing.verify_manifest":{CURRENT_DIR:[9,2,1,""],MAX_CONCURRENT_REQUESTS:[9,2,1,""],async_verify_object_manifest:[9,4,1,""]},"gen3.tools.metadata":{ingest_manifest:[10,3,0,"-"]},"gen3.tools.metadata.ingest_manifest":{COLUMN_TO_USE_AS_GUID:[10,2,1,""],GUID_TYPE_FOR_INDEXED_FILE_OBJECT:[10,2,1,""],GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT:[10,2,1,""],MAX_CONCURRENT_REQUESTS:[10,2,1,""],async_ingest_metadata_manifest:[10,4,1,""],async_query_urls_from_indexd:[10,4,1,""]},"gen3.wss":{Gen3WsStorage:[11,0,1,""]},"gen3.wss.Gen3WsStorage":{copy:[11,1,1,""],download:[11,1,1,""],download_url:[11,1,1,""],ls:[11,1,1,""],ls_path:[11,1,1,""],rm:[11,1,1,""],rm_path:[11,1,1,""],upload:[11,1,1,""],upload_url:[11,1,1,""]},gen3:{tools:[8,3,0,"-"]}},objnames:{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},objtypes:{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},terms:{"0339756":[],"0420947":[],"0938203":[],"0939903":[],"0a80fada010c":9,"0a80fada096c":9,"0a80fada097c":9,"0a80fada098c":9,"0a80fada099c":9,"100":[7,9],"11e9":9,"1394515":[],"1446126":[],"1616009968":[],"1616010780":[],"1616010781":[],"1616018689":[],"1617985382":[],"1617997157":[],"1618604466":[],"1618606085":[],"1619452575":[],"1619452576":[],"1619720217":[],"1619720218":[],"1620328184":[],"1621011995":[],"1621011996":[],"1629133864":[],"1630075391":[],"1630075392":[],"1632510433":[],"1632510434":[],"1632510633":[],"1632510634":[],"1632515171":[9,10],"1731167":[],"2280114":[],"237243":9,"255e396f":9,"2889535":[],"333":5,"3404603":[],"343434344":9,"3514144":[],"363455714":9,"3910377":[],"3926728":[],"39442":[],"4036705":[],"450c":9,"4714":7,"473d83400bc1bc9dc635e334fadd433c":9,"473d83400bc1bc9dc635e334faddd33c":9,"473d83400bc1bc9dc635e334fadde33c":9,"473d83400bc1bc9dc635e334faddf33c":9,"4832823":[],"4904246":[],"543434443":9,"5495481":[],"5964222":[],"641011":[],"6556785":10,"6572418":[],"6f90":7,"7002015":[],"7461076":[],"7649791":[],"7987797":[],"7d3d8d2083b4":9,"8420":7,"8808827":[],"9159722":[],"9335642":[],"934012":[],"93d9af72":9,"9644923":[],"9a07":9,"boolean":3,"class":[0,2,9,11],"default":[0,6,7,9,10],"export":7,"function":[2,3,4,5,8,9,10],"import":9,"int":[3,5,6,7,9,10],"new":[0,3],"public":[3,5],"return":[0,3,4,5,6,7,9],"true":[3,4,5,6,7,9,10],"while":[0,1,3,4,5,6,7,11],But:5,For:[1,5,6,7,8,9],NOT:10,One:6,Such:8,That:3,The:[0,1,2,3,5,7,9],There:9,These:8,Used:9,Will:[4,6],_get_acl_from_row:9,_get_authz_from_row:9,_get_file_name_from_row:9,_get_file_size_from_row:9,_get_guid_for_row:10,_get_guid_from_row:9,_get_md5_from_row:9,_get_urls_from_row:9,_guid_typ:10,_query_for_associated_indexd_record_guid:10,_ssl:[3,4,5],a5c6:9,ab167e49d25b488939b1ede42752458b:3,about:[2,3],abov:9,access:[0,1,3,6],accesstoken:0,acl:[3,9],across:9,action:[8,9],actual:9,add:[3,5],added:3,addit:3,admin:[5,9],admin_endpoint_suffix:5,against:[3,6,7,9,10],algorithm:3,alia:3,aliv:6,all:[3,4,5,6,7,9,10],allow:[7,9,10],along:2,alreadi:8,altern:9,alwai:5,ammount:10,amount:8,ani:[5,9],anoth:5,api:[5,7,9],api_kei:9,appli:6,appropri:11,arbitrari:0,argument:[0,11],arrai:7,asc:6,assign:8,associ:[3,5],assum:9,async:[4,8,9,10],async_cr:5,async_create_record:3,async_download_object_manifest:9,async_get:5,async_get_record:3,async_get_records_on_pag:3,async_get_with_param:3,async_ingest_metadata_manifest:10,async_query_url:3,async_query_urls_from_indexd:10,async_run_job_and_wait:4,async_upd:5,async_update_record:3,async_verify_object_manifest:9,asynchron:[3,4,5],asyncio:[9,10],attach:[3,5],attribut:9,auth:[1,2,3,4,5,6,7,9,10,11],auth_provid:[1,3,4,5,6,7,11],authbas:0,authent:0,authz:[0,3,9],auto:[0,2],automat:0,avail:[1,2],b0f1:9,base:[0,1,3,4,5,6,7,8,9,11],baseid:3,basic:[3,9,10],batch_creat:5,batch_siz:7,behavior:9,belong:7,below:9,blank:3,blob:[5,6],bodi:3,bool:[4,5,7,9,10],broad:8,broken:8,call:11,can:[0,3,4,7,9,10],capabl:8,categori:8,ccle:7,ccle_one_record:7,ccle_sample_nod:7,cdi:6,chang:[3,9],chunk_siz:7,client:3,code:[2,7],column:[9,10],column_to_use_as_guid:10,com:6,comma:9,common:[0,1,3,4,5,6,7,8,9,10,11],commons_url:[9,10],complet:4,complex:6,concat:9,concurr:[9,10],connect:10,consist:3,constructor:0,contain:[0,2,5,7,8,9,10],content:[3,11],control:3,copi:11,coroutin:9,correspond:3,crdc:0,creat:[3,4,5,7,9],create_blank:3,create_index_key_path:5,create_job:4,create_new_vers:3,create_program:7,create_project:7,create_record:3,cred:3,credenti:[0,1,3,4,5,6,7,9,11],csv:[7,9,10],curl:0,current:7,current_dir:9,custom:9,d70b41b9:7,data:[0,3,5,6,7],data_spreadsheet:7,data_typ:6,databas:5,dbgap:10,dcf:7,def:9,defin:[5,7],delai:4,delet:[0,3,5,7],delete_index_key_path:5,delete_nod:7,delete_program:7,delete_project:7,delete_record:[3,7],delimet:[9,10],delimit:9,demograph:7,desir:9,dest_path:11,dest_urlstr:11,dest_w:11,dest_wskei:11,detail:[2,6],determin:[9,10],dev:9,dict:[3,4,5,9,10],dictionari:[3,4,5,6,7],did:3,differ:5,directori:9,disk:11,dispatch:4,dist_resolut:3,distribut:3,doc:6,docstr:2,document:3,doe:[0,10],domain:[9,10],done:4,download:[0,1,2,3,4,5,6,7,8,11],download_manifest:9,download_url:11,e043ab8b77b9:7,each:[3,7,9],effici:8,either:7,elasticsearch:6,els:[0,10],elsewher:10,empti:7,end:5,endpoint:[0,1,3,4,5,6,7,11],entir:7,entri:3,env:0,environ:0,equal:6,error:[9,10],error_nam:9,etc:7,everi:[8,9],exampl:[0,1,3,4,5,6,7,9,11],exclud:3,execut:[6,7],exist:[3,5,8,10],expect:[5,8,9],experi:7,expir:0,export_nod:7,export_record:7,extent:9,f1f8:9,fail:7,fals:[3,5,9],featur:1,fenc:0,field:[3,5,6,9,10],fieldnam:9,file:[0,2,3,4,7,8,9,10,11],file_nam:[3,9],file_s:9,file_st:3,fileformat:7,filenam:[0,7,9,10],fill:10,filter:[5,6],filter_object:6,first:[6,7],flag:9,folder:9,follow:[0,9],form:11,format:[3,5,7,9],from:[0,1,2,3,4,5,6,7,8,9,10,11],func_to_parse_row:[9,10],gen3:[9,10],gen3_api_kei:0,gen3auth:[0,1,3,4,5,6,7,9,10,11],gen3fil:1,gen3index:3,gen3job:4,gen3metadata:5,gen3queri:6,gen3submiss:7,gen3wsstorag:11,gener:[0,1,2,3,4,5,6,7,11],get:[0,1,3,4,5,7,9,10,11],get_access_token:0,get_all_record:3,get_and_verify_fileinfos_from_manifest:9,get_and_verify_fileinfos_from_tsv_manifest:9,get_dictionary_al:7,get_dictionary_nod:7,get_graphql_schema:7,get_guid_from_fil:10,get_index_key_path:5,get_latest_vers:3,get_output:4,get_presigned_url:1,get_program:7,get_project:7,get_project_dictionari:7,get_project_manifest:7,get_record:3,get_record_doc:3,get_records_on_pag:3,get_stat:3,get_statu:4,get_url:3,get_vers:[3,4,5],get_with_param:3,giangb:9,github:[2,6],give:1,given:[0,3,4,5,7,10,11],global:4,good:3,graph:7,graphql:[6,7],graphql_queri:6,group:3,guid:[1,3,5,9,10],guid_exampl:9,guid_for_row:10,guid_from_fil:10,guid_type_for_indexed_file_object:10,guid_type_for_non_indexed_file_object:10,guppi:6,handl:3,has:9,has_vers:3,hash:[3,9],hash_typ:3,have:[5,9],header:9,healthi:[3,4,5],help:9,helper:2,hit:9,how:[7,9],http:[6,9,10],idea:3,identifi:[3,8],idp:0,ids:3,immut:3,implement:0,includ:[0,3],include_additional_column:9,index:[0,2,5,8],index_manifest:9,index_object_manifest:9,indexd:[3,9,10],indexd_field:[9,10],indexd_record_page_s:9,indexed_file_object_guid:10,indic:[0,9],info:[3,9],inform:[2,3],ingest:[2,8],ingest_manifest:10,initi:0,input:[4,9],instal:[0,2],instanc:[1,3,6,7,8],instead:6,integ:[3,7],interact:[1,3,4,5,7,11],interpret:0,introspect:7,involv:8,is_healthi:[3,4,5],is_indexed_file_object:10,its:3,job:2,job_id:4,job_input:4,job_nam:4,json:[0,1,3,4,5,6,7,9,11],just:[5,9,10],jwt:0,kei:[0,3,5,11],kwarg:[4,5],larg:8,latest:3,least:3,librari:9,like:[3,5,8,9,10],limit:[1,3,5,10],linear:4,list:[3,4,5,6,7,9,11],list_job:4,live:[9,10],local:[0,11],lock:10,log:[7,9,10],logic:[5,10],loop:9,ls_path:11,made:3,mai:8,make:[8,9],manag:[1,5],mani:[7,9],manifest:[7,8,9,10],manifest_fil:[9,10],manifest_file_delimit:[9,10],manifest_row_pars:[9,10],map:[0,9],mark:7,master:6,match:[3,5,10],max:5,max_concurrent_request:[9,10],max_tri:7,maximum:[9,10],md5:[3,9],md5_hash:9,mds:[5,10],mean:7,mechan:3,metadata:[2,3,8],metadata_list:5,metadata_sourc:10,metadata_typ:10,metdata:10,method:6,minut:0,mode:6,modul:[2,9],more:[2,5,6,8],most:8,mostli:2,multipl:[7,9],must:5,my_field:6,my_index:6,my_program:6,my_project:6,name:[3,4,7,9,10,11],namespac:10,necessari:[3,5],need:[3,6,9],nest:5,net:9,node:7,node_nam:7,node_typ:7,none:[0,1,3,4,5,6,7,9,10,11],note:[3,9,10],noth:3,now:[1,7],num:5,num_process:9,num_total_fil:9,number:[3,6,7,9,10],object:[1,3,4,5,6,7,8,9,11],off:5,offset:[5,6],old:3,one:[3,5,9],onli:[3,5,6,7],open:[7,9],open_project:7,opt:0,option:[0,1,3,4,5,6,7,9],order:[0,7],ordered_node_list:7,output:[4,5,9,10],output_filenam:[9,10],overrid:[9,10],overwrit:5,page:[0,1,2,3,4,5,6,7,9,11],pagin:3,parallel:9,param:[3,7],paramet:[0,1,3,4,5,6,7,9,10,11],pars:[9,10,11],parser:[9,10],pass:[0,6,7],password:[9,10],path:[0,5,9,11],path_to_manifest:9,pattern:[3,10],pdcdatastor:9,per:[9,10],peregrin:7,persist:8,phs0001:9,phs0002:9,pick:1,pla:9,place:9,planx:9,point:[0,1,3,4,5,6,7,11],popul:10,posit:6,post:[0,9],presign:1,prev_guid:9,previou:[3,9],previous:4,print:7,process:9,processed_fil:9,profil:[0,1,3,4,5,6,7,11],program:[7,9],progress:7,project:[7,9],project_id:[6,7],protocol:1,provid:[0,3,5,6,7,10],put:0,python:[2,8,9],queri:[1,2,3,5,7,10],query_str:6,query_txt:[6,7],query_url:3,quickstart:2,rather:0,raw:[6,9],raw_data_download:6,rbac:3,read:[3,5],readm:2,record:[3,5,6,7,9,10],refresh:0,refresh_access_token:0,refresh_fil:[0,1,3,4,5,6,7,11],refresh_token:0,regist:7,regular:6,relat:8,remov:11,replac:9,replace_url:9,repo:2,repres:[3,5],represent:3,request:[0,3,7,9,10],respect:6,respons:[0,3,4],result:7,retri:7,retriev:[1,7,10],return_full_metadata:5,rev:3,revers:7,revis:3,right:1,rm_path:11,root:[9,10],row:[6,7,9,10],row_offset:7,run:7,safe:9,same:[5,9,11],sampl:7,sandbox:[0,1,3,4,5,6,7,11],schema:7,screen:7,script:2,search:[0,2,3],second:4,see:[6,9],semaphon:10,semaphor:10,separ:9,servic:[1,3,4,5,7,10,11],service_loc:[3,4,5],session:9,set:0,setup:2,sheepdog:7,should:[7,9],shown:9,signpost:3,simpl:3,simpli:9,sinc:3,singl:7,size:[3,9],skip:7,sleep:4,some:[0,2],sort:6,sort_field:6,sort_object:6,sourc:[0,1,2,3,4,5,6,7,9,10,11],space:9,specif:[5,7,9,10],specifi:[0,3,11],spreadsheet:7,src_path:11,src_urlstr:11,src_w:11,src_wskei:11,ssl:[3,4,5],start:[3,4,6,7],statu:4,storag:2,store:3,str:[0,1,3,4,5,6,7,9,10],string:[0,3,5,9,11],strip:9,sub:7,subject:[6,7],submiss:2,submit:7,submit_fil:7,submit_record:7,submitter_id:6,suffici:3,suppli:3,support:[0,1,5,7,9],synchron:9,syntax:6,system:[6,7,8],tab:9,task:8,temporari:9,test1:9,test2:9,test3:9,test4:9,test5:9,test:9,text:[6,7],than:[0,5],thei:0,them:9,thi:[0,1,2,3,4,5,6,7,9,10,11],those:9,thread:9,thread_num:9,threadcontrol:9,through:[7,9],tier:6,time:[1,7,9],tmp_folder:9,token:0,tool:2,total:9,treat:5,tsv:[7,9,10],tupl:[3,9,10],type:[3,4,5,6,7,9,10],unaccess:6,under:[0,7,11],until:4,updat:[3,5,9],update_blank:3,update_record:3,upload:[3,7,11],upload_url:11,url:[1,3,8,9,10,11],urls_metadata:3,usag:9,use:[0,1,3,4,5,6,9,10],used:[5,10],user:[0,10],using:[0,1,3,4,5,6,7,9,11],usual:10,util:8,uuid1:7,uuid2:7,uuid:[3,7],valid:6,valu:[0,3,5,6,9],value_from_indexd:9,value_from_manifest:9,variabl:[0,6,7],variou:2,verbos:[6,7],verif:9,verifi:[2,8],verify_manifest:9,verify_object_manifest:9,version:[3,4,5],vital_statu:6,wait:4,want:[3,7],web:0,what:5,when:[0,3,6,10],where:[3,5,9,10],whether:[3,4,5,7,10],which:7,whose:5,within:[0,2,8],without:[3,5],won:5,work:0,workaround:9,worksheet:7,workspac:[0,2],wrapper:9,write:9,ws_urlstr:11,wskei:11,wss:11,wts:0,xlsx:7,you:[3,7,9]},titles:["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],titleterms:{"class":[1,3,4,5,6,7],auth:0,document:2,download:9,file:1,gen3:[0,1,2,3,4,5,6,7,8,11],helper:0,index:[3,9],indic:2,ingest:10,job:4,metadata:[5,10],queri:6,sdk:2,storag:11,submiss:7,tabl:2,tool:[8,9,10],verifi:9,welcom:2,workspac:11}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index 6e2eda5ce..154d211b0 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -365,7 +365,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1630075391.7987797.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1632515171.237243.log')[source]

Verify all file object records into a manifest csv

Parameters
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index bb23f4915..76ed17e56 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1630075392.1731167.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1632515171.6556785.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters
diff --git a/gen3/cli/configure.py b/gen3/cli/configure.py index c8e39a5c8..3f6399fa8 100644 --- a/gen3/cli/configure.py +++ b/gen3/cli/configure.py @@ -6,16 +6,31 @@ @click.command() @click.option("--profile", help="name of the profile to name for this credentials") @click.option("--cred", help="path to the credentials.json") -def configure(profile, cred): +@click.option( + "--apiendpoint", default="", help="[optional] API endpoint of the commons" +) +@click.option( + "--use-shepherd", + default="", + help="[optional] true/false -- to enable/disable the gen3Sdk to upload using the Gen3 Object Management API (if available)", +) +@click.option( + "--min-shepherd-version", + default="", + help="[optional] Specify the minimum version of the Gen3 Object Management API ", +) +def configure(profile, cred, apiendpoint, use_shepherd, min_shepherd_version): """Command to configure multiple profiles with corresponding credentials - ./gen3 configure --profile= --cred= + ./gen3 configure --profile= --cred= --apiendpoint=https://example.com --use-sheperd= --min-shepherd-version= """ logging.info(f"Configuring profile [ {profile} ] with credentials at {cred}") try: - profile_title, new_lines = config_tool.get_profile_from_creds(profile, cred) + profile_title, new_lines = config_tool.get_profile_from_creds( + profile, cred, apiendpoint, use_shepherd, min_shepherd_version + ) lines = config_tool.get_current_config_lines() config_tool.update_config_lines(lines, profile_title, new_lines) except Exception as e: diff --git a/gen3/configure.py b/gen3/configure.py index cd13978a1..58a769de0 100644 --- a/gen3/configure.py +++ b/gen3/configure.py @@ -29,18 +29,22 @@ CONFIG_FILE_PATH = expanduser("~/.gen3/config") -def get_profile_from_creds(profile, cred): +def get_profile_from_creds( + profile, cred, api_endpoint="", use_sheperd="", min_sheperd_version="" +): with open(expanduser(cred)) as f: creds_from_json = json.load(f) credentials = OrderedDict() credentials["key_id"] = creds_from_json["key_id"] credentials["api_key"] = creds_from_json["api_key"] - credentials["api_endpoint"] = auth_tool.endpoint_from_token( - credentials["api_key"] + credentials["api_endpoint"] = ( + api_endpoint + if api_endpoint != "" + else auth_tool.endpoint_from_token(credentials["api_key"]) ) credentials["access_key"] = auth_tool.get_access_token_with_key(credentials) - credentials["use_shepherd"] = "" - credentials["min_shepherd_version"] = "" + credentials["use_shepherd"] = use_sheperd + credentials["min_shepherd_version"] = min_sheperd_version profile_line = "[" + profile + "]\n" new_lines = [key + "=" + value + "\n" for key, value in credentials.items()] new_lines.append("\n") # Adds an empty line between two profiles. @@ -48,7 +52,7 @@ def get_profile_from_creds(profile, cred): def get_current_config_lines(): - """ Read lines from the config file if exists in ~/.gen3 folder, else create new config file """ + """Read lines from the config file if exists in ~/.gen3 folder, else create new config file""" try: with open(CONFIG_FILE_PATH) as configFile: logging.info(f"Reading existing config file at {CONFIG_FILE_PATH}") @@ -60,17 +64,18 @@ def get_current_config_lines(): def update_config_lines(lines, profile_title, new_lines): - """ Update config file contents with the new profile values """ + """Update config file contents with the new profile values""" if profile_title in lines: profile_line_index = lines.index(profile_title) next_profile_index = len(lines) - for i in range(profile_line_index, len(lines)): + for i in range(profile_line_index + 1, len(lines)): if lines[i][0] == "[": next_profile_index = i break del lines[profile_line_index:next_profile_index] + lines.append(profile_title) + lines += new_lines - with open(CONFIG_FILE_PATH, "a+") as configFile: - configFile.write(profile_title) - configFile.writelines(new_lines) + with open(CONFIG_FILE_PATH, "w+") as configFile: + configFile.writelines(lines) diff --git a/tests/test_configure.py b/tests/test_configure.py index beb92f317..92e29367b 100644 --- a/tests/test_configure.py +++ b/tests/test_configure.py @@ -42,8 +42,6 @@ def mock_access_key(_): def test_get_profile_from_creds(monkeypatch): test_file_name = str(uuid.uuid4()) + ".json" try: - profile = "DummyProfile" - creds = {"key_id": "1234", "api_key": "abc"} with open(test_file_name, "w+") as cred_file: json.dump(creds, cred_file) @@ -63,8 +61,12 @@ def test_get_profile_from_creds(monkeypatch): def test_update_config_lines(test_lines, monkeypatch): file_name = str(uuid.uuid4()) monkeypatch.setattr(config_tool, "CONFIG_FILE_PATH", file_name) + with open(file_name, "w") as f: + f.writelines(test_lines) try: - config_tool.update_config_lines(test_lines, expected_profile_line, new_lines) + config_tool.update_config_lines( + config_tool.get_current_config_lines(), expected_profile_line, new_lines + ) with (open(file_name, "r")) as f: assert f.readlines() == [expected_profile_line] + new_lines finally: