From 096ca82be438b475b116de5bf9ac930201f2ae5f Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Tue, 17 Dec 2024 15:22:07 -0600 Subject: [PATCH 1/8] Add post_indexing_validation module --- gen3/tools/indexing/post_indexing_validation.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 gen3/tools/indexing/post_indexing_validation.py diff --git a/gen3/tools/indexing/post_indexing_validation.py b/gen3/tools/indexing/post_indexing_validation.py new file mode 100644 index 000000000..e69de29bb From 0dcb2ca7926200c7c749dce4b28ee0f47293626b Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Tue, 17 Dec 2024 16:25:47 -0600 Subject: [PATCH 2/8] Add tests --- .../indexing/post_indexing_validation.py | 250 ++++++++++++++++++ tests/test_post_indexing_validation.py | 174 ++++++++++++ 2 files changed, 424 insertions(+) create mode 100644 tests/test_post_indexing_validation.py diff --git a/gen3/tools/indexing/post_indexing_validation.py b/gen3/tools/indexing/post_indexing_validation.py index e69de29bb..3b3d1d32a 100644 --- a/gen3/tools/indexing/post_indexing_validation.py +++ b/gen3/tools/indexing/post_indexing_validation.py @@ -0,0 +1,250 @@ +""" +Module for running post-indexing validation. +validate_manifest takes as input a manifest, an api credentials file, and an output path +then attemps to obtain a pre-signed url and download a file from each bucket +to verify it has been indexed, and then generate a report in csv format. + +The output format is as follows: +| ACL | Bucket | Protocol | Presigned URL Status | Download Status | GUID | +""" + + +import csv +from cdislogging import get_logger, get_stream_handler +from gen3.file import Gen3File +import requests + + +logger = get_logger(__name__) +logger.addHandler(get_stream_handler()) +logger.setLevel("INFO") + + +class GuidError(Exception): + pass + + +class Record: + def __init__(self, guid, bucket, protocol, acl, access_token, commons, size): + self.guid = guid + self.bucket = bucket + self.protocol = protocol + self.acl = acl + self.commons = commons + self.size = size + self.response_status = -1 + self.download_status = -1 + self.headers = { + "accept": "application/json", + "authorization": f"bearer {access_token}", + } + + def check_record(self, gen3file): + """ + Checks the status of a record by generating a pre-signed URL and attempting to download the file. + + This method performs the following actions: + 1. Attempts to generate a pre-signed URL for the record identified by `self.guid` using the provided `gen3file` object. + 2. Logs the result of the pre-signed URL generation, including the response status code. + 3. If the URL is successfully generated (status code 200), it attempts to download the file from the generated URL. + 4. Logs the result of the download attempt, including the status code. + 5. Sets the `response_status` attribute to indicate the success or failure of the pre-signed URL generation. + 6. Sets the `download_status` attribute to indicate the success or failure of the download attempt (if applicable). + + Args: + gen3file (object): An object that provides the `get_presigned_url` method to generate a pre-signed URL for the record. + + Returns: + None: This function does not return any value. It modifies the `response_status` and `download_status` attributes. + """ + logger.info(f"Checking record {self.guid}") + try: + resp = gen3file.get_presigned_url(self.guid) + url = resp.get("url") + response_status = 200 + logger.info( + f"Pre-signed url successfully generated for record {self.guid} with status code {response_status}" + ) + except requests.HTTPError as err: + response_status = err.response.status_code + logger.info(f"Pre-signed url generation failed for record {self.guid}") + self.response_status = response_status + + download_success = -1 + if response_status == 200: + try: + download_success = requests.get(url).status_code + logger.info( + f"Download process complete with status code {download_success}" + ) + except: + download_success = -1 + self.download_status = download_success + return + + +class Records: + def __init__(self, auth): + self.auth = auth + self.access_token = auth.get_access_token() + self.commons = auth.endpoint + self.record_dict = {} + self.record_sizes = {} + self.headers = { + "accept": "application/json", + "authorization": f"bearer {self.access_token}", + } + + def read_records_from_manifest(self, manifest): + """ + Parses a manifest and creates a dictionary of Record objects. + + Args: + manifest (str): the location of a manifest file + """ + if manifest[-3:] == "tsv": + sep = "\t" + else: + sep = "," + with open(manifest, mode="r") as f: + csv_reader = csv.DictReader(f, delimiter=sep) + rows = [row for row in csv_reader] + + try: + guid_cols = {"GUID", "guid", "id"} + guid_col = list(guid_cols.intersection(set(csv_reader.fieldnames)))[0] + except IndexError: + raise GuidError( + "Manifest file has no column named 'GUID', 'guid', or 'id'" + ) + + for row in rows: + url_parsed = False + size = row["size"] + guid = row[guid_col] + for acl in row["acl"].split(" "): + if acl != "admin": + for url in row["url"].split(" "): + if "://" not in url: + continue + else: + protocol, bucket = ( + url.split("://")[0].replace("[", ""), + url.split("/")[2], + ) + key = (bucket, protocol, acl) + if key not in self.record_dict or ( + int(self.record_dict[key].size) >= int(size) + and int(size) != 0 + ): + record = Record( + guid, + bucket, + protocol, + acl, + self.access_token, + self.commons, + size, + ) + self.record_dict[key] = record + url_parsed = True + + if url_parsed == False: + logger.warning(f"No url parsed for record {guid}") + + def check_records(self): + """ + Iterates through all records in `self.record_dict` and checks each record's status. + + This method performs the following actions: + 1. Initializes a `Gen3File` object using the authentication information from `self.auth`. + 2. Iterates over the items in `self.record_dict`, where each item consists of a tuple `(bucket, protocol, acl)` and a `record` object. + 3. For each record, the `check_record` method is called, which attempts to generate a pre-signed URL and check the download status. + + Args: + None: This method does not take any arguments beyond `self`. + + Returns: + None: This function does not return any value. It triggers the `check_record` method for each record in `self.record_dict`. + """ + gen3file = Gen3File(self.auth) + for (bucket, protocol, acl), record in self.record_dict.items(): + record.check_record(gen3file) + + def save_download_check_results_to_csv(self, csv_filename): + """ + Generates results from presigned url generation and file downloads. + Output format is: | ACL | Bucket | Protocol | Presigned URL Status | Download Status | GUID | + + Args: + csv_filename (str): the relative file path of the output csv + """ + download_results = [] + for record in self.record_dict.values(): + download_results.append( + { + "acl": record.acl, + "bucket": record.bucket, + "protocol": record.protocol, + "presigned_url_status": record.response_status, + "download_status": record.download_status, + "guid": record.guid, + } + ) + + self.download_results = download_results + + # Check if the results list is empty + if not download_results: + logger.warning("No results to save.") + return + + # Define the CSV file header + fieldnames = [ + "acl", + "bucket", + "protocol", + "presigned_url_status", + "download_status", + "guid", + ] + + with open(csv_filename, mode="w", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + + # Write the header row + writer.writeheader() + + # Iterate through the DownloadCheckResult instances and write each row + for result in download_results: + writer.writerow( + { + "ACL": result["acl"], + "Bucket": result["bucket"], + "Protocol": result["protocol"], + "Presigned URL Status": result["presigned_url_status"], + "Download Status": result["download_status"], + "GUID": result["guid"], + } + ) + + logger.info(f"Results saved to {csv_filename}") + + +def validate_manifest(MANIFEST, auth, output_file="results.csv"): + """ + Takes as input a manifest location, a Gen3Auth instance, and an output file + Attempts to obtain a presigned url from a record from each bucket then download the file. + Outputs report in csv format. + + Args: + MANIFEST (str): the location of a manifest file + api_key (str): the location of an api credentials file + auth (str): a Gen3Auth instance + """ + logger.info("Starting...") + records = Records(auth) + records.read_records_from_manifest(MANIFEST) + records.check_records() + records.save_download_check_results_to_csv(output_file) + return records diff --git a/tests/test_post_indexing_validation.py b/tests/test_post_indexing_validation.py new file mode 100644 index 000000000..dbea6c91a --- /dev/null +++ b/tests/test_post_indexing_validation.py @@ -0,0 +1,174 @@ +import pytest +import os +from gen3.tools.indexing.post_indexing_validation import ( + Record, + Records, + validate_manifest, +) +from unittest import mock +from unittest.mock import MagicMock, AsyncMock, mock_open +from aiohttp import ClientResponse +import gen3 +import time +import base64 +from requests import HTTPError + +cwd = os.path.dirname(os.path.realpath(__file__)) + + +read_data = "guid,md5,size,acl,url\n255e396f-f1f8-11e9-9a07-0a80fada099c,473d83400bc1bc9dc635e334faddf33d,363455714,['Open'],['s3://pdcdatastore/test1.raw']\n255e396f-f1f8-11e9-9a07-0a80fada098d,473d83400bc1bc9dc635e334faddd33c,343434344,['Open'],['s3://pdcdatastore/test2.raw']\n255e396f-f1f8-11e9-9a07-0a80fada097c,473d83400bc1bc9dc635e334fadd433c,543434443,['phs0001'],['s3://pdcdatastore/test3.raw']" + + +class MockResponse: + def __init__(self, status, ok=True, json={"url": "my_presigned_url"}): + self.status_code = status + self.ok = ok + self.json_data = json + + def raise_for_status(response): + pass + + def text(self): + return self._text + + def read(self): + pass + + def json(self): + return self.json_data + + +def test_validate_manifest_coro_with_200(): + hostname = "test.datacommons.io" + exp = time.time() + 300 + decoded_info = {"aud": "123", "exp": exp, "iss": f"http://{hostname}"} + test_key = { + "api_key": "whatever." # pragma: allowlist secret + + base64.urlsafe_b64encode( + ('{"iss": "http://%s", "exp": %d }' % (hostname, exp)).encode("utf-8") + ).decode("utf-8") + + ".whatever" + } + get_mock = MockResponse(200) + with mock.patch( + "gen3.auth.get_access_token_with_key" + ) as mock_access_token, mock.patch( + "gen3.auth.Gen3Auth._write_to_file" + ) as mock_write_to_file, mock.patch( + "gen3.auth.decode_token" + ) as mock_decode_token, mock.patch( + "requests.get" + ) as mock_request, mock.patch( + "csv.DictWriter.writerow" + ) as mock_writerow, mock.patch( + "csv.DictWriter.writeheader" + ) as mock_writeheader, mock.patch( + "builtins.open", mock_open(read_data=read_data) + ) as mock_file_open: + mock_access_token.return_value = "new_access_token" + mock_write_to_file().return_value = True + mock_decode_token.return_value = decoded_info + mock_request.return_value = get_mock + mock_writerow.return_value = None + mock_writeheader.return_value = None + + input = f"{cwd}/test_data/manifest3.csv" + output = f"{cwd}/test_data/post_indexing_output.csv" + + auth = gen3.auth.Gen3Auth(refresh_token=test_key) + auth.get_access_token() + records = validate_manifest(input, auth, output) + + mock_file_open.assert_called_with( + f"{cwd}/test_data/post_indexing_output.csv", mode="w", newline="" + ) + mock_writeheader.assert_called_once() + mock_writerow.assert_called_with( + { + "ACL": "['phs0001']", + "Bucket": "pdcdatastore", + "Protocol": "'s3", + "Presigned URL Status": 200, + "Download Status": 200, + "GUID": "255e396f-f1f8-11e9-9a07-0a80fada097c", + } + ) + + presigned_statuses = [] + download_statuses = [] + for record in records.download_results: + presigned_statuses.append(record["presigned_url_status"]) + download_statuses.append(record["download_status"]) + assert set(presigned_statuses) == {200} and set(download_statuses) == {200} + + +def test_validate_manifest_coro_with_401(): + hostname = "test.datacommons.io" + exp = time.time() + 300 + decoded_info = {"aud": "123", "exp": exp, "iss": f"http://{hostname}"} + test_key = { + "api_key": "whatever." # pragma: allowlist secret + + base64.urlsafe_b64encode( + ('{"iss": "http://%s", "exp": %d }' % (hostname, exp)).encode("utf-8") + ).decode("utf-8") + + ".whatever" + } + mock_response = MagicMock() + mock_response.status_code = 401 # The status code you want to return + mock_response.url = "https://exampleurl.com" + get_mock = MockResponse(401) + with mock.patch( + "gen3.auth.get_access_token_with_key" + ) as mock_access_token, mock.patch( + "gen3.auth.Gen3Auth._write_to_file" + ) as mock_write_to_file, mock.patch( + "gen3.auth.decode_token" + ) as mock_decode_token, mock.patch( + "requests.get" + ) as mock_request, mock.patch( + "csv.DictWriter.writerow" + ) as mock_writerow, mock.patch( + "csv.DictWriter.writeheader" + ) as mock_writeheader, mock.patch( + "builtins.open", mock_open(read_data=read_data) + ) as mock_file_open, mock.patch( + "gen3.file.Gen3File.get_presigned_url" + ) as mock_gen3file_get_presigned_url: + mock_access_token.return_value = "new_access_token" + mock_write_to_file().return_value = True + mock_decode_token.return_value = decoded_info + mock_request.return_value = get_mock + mock_writerow.return_value = None + mock_writeheader.return_value = None + mock_gen3file_get_presigned_url.side_effect = HTTPError( + "An error occurred", response=mock_response + ) + + input = f"{cwd}/test_data/manifest3.csv" + output = f"{cwd}/test_data/post_indexing_output.csv" + + auth = gen3.auth.Gen3Auth(refresh_token=test_key) + auth.get_access_token() + records = validate_manifest(input, auth, output) + + mock_file_open.assert_called_with( + f"{cwd}/test_data/post_indexing_output.csv", mode="w", newline="" + ) + mock_writeheader.assert_called_once() + mock_writerow.assert_called_with( + { + "ACL": "['phs0001']", + "Bucket": "pdcdatastore", + "Protocol": "'s3", + "Presigned URL Status": 401, + "Download Status": -1, + "GUID": "255e396f-f1f8-11e9-9a07-0a80fada097c", + } + ) + + presigned_statuses = [] + download_statuses = [] + for record in records.download_results: + presigned_statuses.append(record["presigned_url_status"]) + download_statuses.append(record["download_status"]) + assert set(presigned_statuses) == {401} and set(download_statuses) == {-1} From f7230aa863077f868a373e6733a606b8d3c1a709 Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Tue, 17 Dec 2024 22:27:07 +0000 Subject: [PATCH 3/8] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 447288 -> 447288 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 100192 -> 100197 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 35909 -> 35909 bytes docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index 8d2a91031f0341fe29cc971c863135396ddfab0b..775aba316cac9dc6e4125697db58b7f2c5f39d55 100644 GIT binary patch delta 7723 zcma)Bc|ep$vbURW=HtFahZ}|{h~VG_id-s!f{0g)hz=+OoFxnqNS-;w9Gal>0@}oS z*{l&ao3M%ToW#!q6_aQVuV|7@jH@x5-Q<~=C`!Ns%~p5MfbRR}{Xgej~>gxWO zJ#U8Wc{60kH*pQRN9)U%EpM^u>a5nP+7_F)7)Lx(URPRHT36a)bFZym{K(SU+NIX& z7Mt?$(yDSCO6%(?TWl_m*P?}Mzplf0X?0ooe=V)9Kv_SjSgect{g*APHP_TvRr$5p z^w!0e@+EZ`f>36bVKJneH4Td-ciE0%#_HR2zyO=4bCSemwRJ6)8ys_&h7!5M5jU|I z33PJ6yFYk|b_q4HMhRTwfQk!|Y!AzdH;`VoAl?vilZ*WC&xh2y&YI#w8oE(#Aujmm zH%?Jb-Q|=Er*3gd;S^thMaL-~wnfh=9Ltx-*llH*aWQHyrID`Y&f zHR}AD_XNkITcdvQ?HDoYxxvnx!pR-hYYHWu%#aj9y4aYcNOFxmm=uG%_GD5L`@1R3$L8y_l_j-hsiMz(2;o~0$vW1GesnU-9NF}IG7W~nJ=wlUSn z4x}t%o5vYh`nV-59Ql>M=wL+x31EL8m&pQ?jVxn)32R9-xjf8;NRvD*LV!z_xWnWo;Q=_ST#?{aNg_FjzT? z1Vd&58Hafu&3-D%WzAh;3ijZF_Z#@qZEAUS-Lj?SwJi%kR| zIpJ^L`1%b=w7qdz5UlNufl!i675l-q&+kJ#U~DGwvw!hcyMUqO7^e5FEkE=tolmA7 zY|Hn4_A^`e3Do}Db_0d#24cmkjbb^wa-n7u`e@y?ig3!`p8RfygL2#5BPm$2|PU-`+VI%B*|!_G$ahNtjtXi~=cX|IVS<>VtMg2VJOiZSBB9;FGY(msNY?`%d?w zEMEEW);;d+?;O3;@78|dW2Z~U+(FRPNxba0KRxcy+O|x!Up(IF;5NRlvf*C^?K<_7 z01Ry2`JnZuRD0xUXRxTxE*=kpm0d({H+}xOG!!Li5B<^+8ACs>5C2L8EqrkGAU5K( zpZ(dhdmJV_>*(KLuR3=|LiWP*R4x>DG+_q*@a=6WV1?(SU`IN2Wy$OO?7O}@5H*&x-cmE(c`=d87 z2%%MYUbR1YE0ltJ0=i4b{uXy97tSV-82ih2>IkR&zuK6uMddrrOAE)JUa zE3O)P{9C`XI5^}^RLFB7{+iu`eCUu|9EMSR=81kTJ?5`{;^p+CVR(vTDx5VMd;^H8 zo%bW(+-r!Sm(~+Nuz*BR`Oo1x6M{)NG_@-pT3`_QR!THwF!6P`<4PJTVcm;G;Yz$5 z zvE)mK4lZ^W1>bGBQI5-J0i^Gi8xMC9$nJ5XUj?cI28c$=uL0f&`6a+dejQ>%v_wh; zc}<8L;<|P@k<1}%#wIIlH<3!#yhZdfEj5WeMv%jDI;+B#CrK#y&cr0O>Zq#yHj-#U zs2vO;p2#RKMTkX0rM)IFZ)ZjpSC`gd?4&$?Q&x5bL-ZOAJxEDY@6PUfI4l)X{xSy!R;!T-JuqWRu zb=NXylg}iE5yWqpfx4LmxUuHB{|8^;jrfXrItEt4J*ry^e zKuiaw0IJj0TF6l$?ceK7{+O1gDreA(!toG2QSA&)tU;(8s3B*iV{DFIuyq;<0xkgG z+35J}GIB~vF|WoxfdZbV6gSu#iAP#jPx9r)EP0#+NebPoa(P&L8rI(L(#f{BoKyJD z!5#W&4h_<7Jx%He({@_<>#J=29$CHRD@g!hD?bd>Y|oJGWc?n0ZTfTAf;dsS3M-Qn z&%A&~8412>?bSv+rX1a|mLv+|>VK1M!V*_@SrB+#{fiU)VtGLOl#xT6!+Sslt%-zd z9WP5n%69|l=-1x*3OS6NL3`m<@=qR`LD<0f2eR;zHqLHdTJ-z@YrxOOj-cfmdEnq7Gk21 zX(CqsKrI8`xw4CB>UQEwIO`ASHc(63fk#L3c(T1<;cj98Q!!@TviXGXaKb_H z>ns?)J*{`umY?D$9v!pRt*)B+H2Fz#TbH>)#W766@z3-gTEpkq{EPXsG*5o0<++L{ zdma^U)4T}rG|h7pFVnnG@i5JE5%1DGop_e!4HB=?JR*8--UFf~Bgi{4I_rU-za*0i z_*)no_dUvZcdQetZv0ozyKW$5*>k z&vI9*{E950q+2_FmQ*O@t~UAtS?57+Yez4UeU#kLp1Dk>d5}AByobCCgNEYSdFck3 zM@X0U!%Y(4POiZ%7n(+TwG@Tk4j?zR(SzwecXD035=~~y2p=ti;6jbfV{Mf8ng7aGO-8pRej;tqSq(_+#Me~71JarkpQourEY zMRX2@%*k{RxSb=h-j-p*1`Kc%K-W+xNku{7a5@Z^`{Z!iFY(#obe` zNQlvgZo*T9@g6UB5FKGwVSO$Y9& z!bMRk&5275IyYp|DUe~t z%P%hr`7T(UMf3IIi}ec6@kstK_f=);yC7;JorX$P6KNg}dnVH9Quj93oe&`Yk>^N{ z@NrjcM3Q({_hq{aHcz6{(f-OLI!)TEusH|k;;(2}F`2qb)E$zt=~(fCh7*&i2b!Fn zOlRRRJe$tMVMVrx zo2Jo`C^$2X&Xv~ruz5PxxV3-|6K_mdmPZ4mXoeKE=Fuz(%#uKF9&T4JjLN4EO8g;- z|0SPJl)wTBB+Q^A_(RhdwX>$vA{_pNgVZULIymtN{)7yaXhk17NdguLoG75#5_qJ~ zcLw@hwSD*q@GTT`MitUIa{0^Sz$Xot|AHTVz{O#xv;z2|U{e z;8Pjjiv321XVTeH=S8VAVirxuNUCR1GY%(b(E=R8X482%)X%17sSL1g5e&I#uHD_u(;6`H)llh(wzn!Y+6dIv&EhzX_i6v6J0|;`x{l5&v!|%B8^&8i~CH zAAkce#7EW%@j;v%{Thk^xk3JxzXpv6-8d#UVEjP3k76cp1+ zJ-=%$zsNXF7o%4JT$MojLd>`Tu1jF!LOL7{6=JtJxsVnK3U4s6e-jsB$GQ%-MKp!< zz>!7RjCwYhX$CTxW^7+Q@VuFh!{MMAtD*<)NG`dArmNz=kp9Eq&{(X*8*$iYEwW(= z!N6f~a108{K9H|4Pq=K;90b4*0{@Z&R|F3sNd7ZhouC%Vcwne-a zY=QTQ9xGoaewVS_h7PnwC+C`74ETWgt=Z+FCcFj;B9%xStdUBrxGroFENNc=G>0qR z2p{T$gE+h)O3_KJaG|wrJoS=31~a@YUGQfY?uj!#s(7~Ga=-gDSSM)63R4D2^AI?E zjRZnRxZ);(XbxZ|ZouzRO0dKXtndqi@R=H|MB_Y*qLnxUzrAg~e38=~zd-RE@R?hq zl?c@8h*pN;fLsv26CD!_Vjce&6z2F{$p1cb`TG_+$TBLyXjyFJvuO=RWr>@fKSphX zq2vZJP|F?7B|$kEuIF!3+aT7^J(yZLp)*ejQaA)F^Du%=Ek0k_SFP~d4YD*Ps*eg$ zBKOH< z>8o{NGLX#Yby{L%ykpkd9)O)8G!UDui zUOKHeNNQ&vb_%gbVIx|{yxQ4y)@7v?<%#t*G81Bg_RsX8L7b|$xmMueOXR=o*0t2v HFHZb7O-GK9 delta 7776 zcma)BdtB7jws&uSGe6#M!Ql;qilPXvb&MrJ;zex|0iB#yWz4y$Z_x^MLnZ4F`t-bczYp>th z>o>n6Zw4QEGx)$)!|I%#tgddhc-ENB71g6wm(~=Q6xS3xfRZ)Ug;li`70ni%&}glRwBQ0 z#6?(;KpzJ@``wgiw@?#%k-!}es2BvvcChqV1LqWVV zIN{G-PEk(X=adtt?r}=t6yJbJ$0_c%NzWIOWEv zE>5{~iu+~qa8mHM(>iiQl9TYHL0!GEWlP{Nz3`?%_4xh`!Euio)V%wW<6bqWG0|@c z8TYI~{ryMDaqk+`Z&$t}WZc6B^`&Ey<6btX|6I3M$hfBsYWB;g1b1hOV87~=EO)d) z9lPR;5OQZ5)B&%Y65<%}*M>1^)kaV%ntXHbCMxIEt4n(_*1|gL*~%TT zI3S_V-Wr|m)MV;pdlN$0(J>=f?&xK#b&M}!9jtH69@1-ld+a>I{1c5VeEeM&GSSF( zjMKA}#6@fe(pq9M+n#J>$w`Im!8jv3o>YLlX_*+sQYWln!4r&Z{VzILH z!ccYc#3I(5XoPEZB!pd`n8tQaFhbovWR@kTu~(9etUb9_CgyscDk3po=9EaxH#sE| z)5lt|1UIVBSF4yQz75}tV?9X&ZE(lJeGtDukUh9QPM#36zMqB+;pW=j%b;o3@Bvz#VA_#W-d1q_+)O&(~8^ejS z^^ep0q$uL3Gi#mfVQtBH)0gG!2!-`yND!orBoi>eW7*GnnXGA#4{JH(#~xeyejVSl zMJ=tYSyNtG-Mp+hCfmfH*-g#aA~1a9!Qdkg1s{1J_{hV+M;-(|@(}Qm2Y`=!f18?1 zE$-!&D=KPBN_kZjONbR@+@nRLMUX{&MQBA-WkAt{sl$3K-;JWzzSe&fJedY*@x*LP zl0bHKM<#eLBw|C>jFw8mIbYT?S+!o=d0cYV?XUOPsMcLKZ5pk+20~FHRjemkKYJ8% zgYjv^$NKs1Hi<*gIigy3@A=^&wQGsgjqUl)$NK5sqY}3MWxs(!WgYsaB}TH0Lzz&u z1I@G?+DJI%YfXH&-A1__{#8=2{#7(>IMQun_iZ66d*@6DGry;|&N_Nk!pz)m6i89q zH#Wr?Un%2S8&7 z@vz=Md)}tCZ;fibalXUGZGT;5Bfbncbm?aS7}%m~0b4Jr)`-h?XHlQsI3ED(?-0E; z{xb|N-#u{i){rl3o-wrZ`iL)u)4~Q<4`8D%`&gg7a>O=;#~tk(tQA*3m5?>}8kHM` z1C40!4`1Jx0+xF%5)PzNXO_6d$9m|SlQ#Y5zU#74FMfZ^rphTg%{6ZRarW$OkwP=otf-R zfy~9oE;B*Km!&Spa8IxcdD;QLTL3uN0NU>V;sUHh=lNCiYpwrM|9BOLVUF8gq(g<>ZQ$gBA*G2 zt*=m_&4&bSdqCW@WjgYOR80#ZkqCLP4|e-#bth=HbKinE5M|L(9F11DXH3RzCX zSF^g252b*on%3k&uneJZ8v5o$ImOr1aLxMFx#)3rBLK13lH>;B!?%6!tbvR z7$B-BFaEp=^3u;oeyw9QG(|`Sc{zxCp;NmVM;2fe*+#qNA#IcQnzYR68-21XH1?{cAjiMUZf*jd*Bb6Nu+SJRm;=3ODM! z*&EY+w5`eH18D;`uO&u@b!Rv*33Fsu3YjA(V-Zp{w={Cmrd_v`#KNdFqSr#Ekbl_R z&`%?7HaCnj$VS_;yjVf&EE1#rlu35kR9j{deBk)jv}3c0vrQGP#zXuZ>Y{n(;DO0) zplUo6dQcrq&c!@`Y8B>r(p-Y&nNJ71*i}oLPd>4M=t=Ch9Mn$B#?voc_*vfNE#gy1>7$wF7RFio{+AV z%(flYiYG~cjl_tlww+>jYM7nB+YIg7ZC}TG0Uq1m7SI6g-ZKPCnRd|3JF}9lM?{&_ zO4k!V!q$K2uUR&b{bcJAUkz&lrJN|`%4m$^20#d`?dGJOim$Z&|Y|z{FA$8 z5OZK+ICa(D*^c$j<_VxDUQm@sz2M3c>Z$#82l<%GG(2ZTIQHi2#3+cXyX4;ZLj>Ap zoE8y9z2Ryeb=NMo;xomkVbJ`zwtQ@F>>=?KTt{L}z~;3Cu8btnn!2BO6HfZTSG&kS zE#&}zo&^8%c;X4m4if{!=VP?v5AlbROEkl~WH6Ov&S73Ag#O;$cKxdh=^(hd8#T`z z;dv$|y>`^0>5Tz%c=mv{~5rWRi& zcy%Jh@G^(8b65q=f1-EO>OLdhG@m!HS@In(%UN{wSyXhfS>d9I&2kYvY*vV9VY8e> z2b-l64Qy7R=wGvlD9~AhMg2yQmv1!I0ss1fOwHz9GgkSB8rlC(AB&Spg>^nvyd22- z4}bhS2@!83_(~@HZ4%vaKgX^2bGElXn+ys=V`)C=g+Ij7@i@E|OQ)*hZ;XzikT#75g3DDB<7pZ`e82!(0^Au2 zMad}09YKfVcAp+W@3@c$P&J-r;_%*hnxYqPIMr}sJRKt5I@pO;?0XWaDkV=iB)$?@ z3$xSlsccH5krMc$)WG)+U$l8Ukxs&4WD=c7euGs>bP5i8l4zD*yhT~yKoT7Y`xj9? zT%SY(Sk(zVe}V99wz(~|8o?`>j^#Mq4pUs+<&K1CD;%GK%kQ5++{c9IkLOnMO())8={TI4L^E)3Nue`wn4dy(<-|4_s%@#%@aX{G^BVlc_T*)K3;pd^QFnPNQQdZ(m6Q))>Jx6 z&R1b)2Cn7phj;S2Y1CPwu8=UDju&k=T$o1PaLScwbRG^Prqj7NteY8tHE%NMFLADFfxaZ;w`8*YNyYk zc{uzT2dPscb#UQPytDL|Xqf|@Dgl!OE@ac`5?Jl9JsIt-&asD~+JTP(?_9BFWG-DG zx4(86c%|U>U+|%W?N3D)wCB=!(rBG+@?1Je0?#@CY@)Go?4NXME}buRUX(hc=3z#5 zLFGJJh{MHsG#iJ|`E(HuwexAAR0h~mK>gw0^X>b1MFMjc&;*#z-*yORNbS86xH^}*Lt8L)*=C@=a3YAh+4s@|Igepo z?ScP#j26i`R=K3_BAO_H_oNZ5f6?&bBI+*DHU}Dy({C)IGbR4M1CNGPi|yK{CEBz3ErHZ!7;yn~ zNnrajIs#{&T}BHe-YqRoDZpCQ36=tyL^|Mf0hXnXZH06)GHHcaxH{nZLOKD56NQ)( z9ndSe#3GuiioYm*hr`M7n1(&Wu)>;TwGx7X!{Nj@6qI}*`!9F6RYdh@qPvJDhl|(Y z_+bMEtQxawOnpuBhGx^JvACh%NAQcY3v!ETC`O=C4i-7Q&4*6yqhk72IbObxpTkFt zU%nOt+c4eW+$=gSOB6CjYU+i{=O}&%(YnvkaUY6%(h?3^#?wfJ-?uQb(J2sb$;u|D zpd=1h{4(&r@>%?^^0LyJ!s?phn%e4S(I;5^9}zuKx<-7qF+G5GoQ+1VHaQuvb-CN* zw73y3fb0k*0ta)15+g1OOE^n7>IY3>iYLM+9dH1Lw?!&CsTC%)_D!T7(#9Z$*Q67E ztKpV7z} zloEyO6htY*4E)}x4i@i6T7HemSoEct(_3%MkcxEE%d3O)T_2s|!*1tqB*O z)fuHiI20$72#OEe;~RBRv0Q5LoMwKqvcg!PG1!GQk**wD!5b$)2M zZIdn%Tl?#qbSrXIUM{OGgW+H>^~Z|uqpki@H+w-Chb`VEr4^+$rGk9^3&BdCl_oI**?L6aW*%j-Sm1NMjatcMUyWc c5QfT1W6Dm+V2_=WA=U%Yj^+eLU&a(G0M++0KmY&$ diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index 82870e4ad9277d8ad014c270a6abbd8fde3e7694..edadcb8be2ad0c450d3f771c32de312862795d8f 100644 GIT binary patch delta 111 zcmX>)gX!oDrVal!`Ap1BOpJ{5ER4;}%+1UuC+OR3_SE8JMHAngpud{~O>DBJg&(?5 OtA#U~(B^9v`N061k{u@i delta 111 zcmX>)gX!oDrVal!`3#H=4b05-jDVDhf$8J~eVfgmTAZwC;+qrncXObLP1dyVLlIndexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1734031066.998105.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1734474422.6146133.log')[source]

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index 98305b101..3f9430065 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1734031067.2310405.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1734474422.8366766.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters:
From 159989b681a06f6abae60d052e9a3bd600870a11 Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Tue, 17 Dec 2024 22:28:58 +0000 Subject: [PATCH 4/8] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 447288 -> 447288 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 100197 -> 100192 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 35909 -> 35909 bytes docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index 775aba316cac9dc6e4125697db58b7f2c5f39d55..d33c9f20808228662cabf018b2ce654fcb95d7b2 100644 GIT binary patch delta 9199 zcmb7KX&e@?Wy{T4I8lo zVX?MqwXajjfiUan=*^GlkRdh~9!$c9+PYTjW6rw@6$yOD0S||b#QKF1ACKfagpQ+g ziFI52rjh#pAMiYP9&bSBZ~H(}+l#40#>}*EWsPi)fk*sdaM& z$0KSoknNJ=Q8gK2-&-JLJhCQ({@I0sS8a6HN;gUM~F;CQ4> zhKM6If_suJ*nQKV5-bn2$>4grUT{3vCd1+6=LFX)xG3-E1;+>1WSDyE`D<*rC67E} zHI`uVm~FQNr|~$OTVrfK2?>d-E2@$cW!BVeY;6(q$z|RSnMf8P>SVVpVXQydg=vz8 zl0Ifk3L)KWY*GYy!WJe)1fYB@vV8&TMeV(nvwG4q%J5-X@DNkfBLtb88+dwAR{JDxm+ zbg_ZteK;TQrd-4MXqynlc8^`gc26*~@Nso4D9y}{j$6)tf$i?GdX|y8h=q(dGc~o0 z?MgSZ^t2LYLfuno%UNrxnPrY&iNoh!8otrNrg-Ad{ysj7v2kWLX+kOM8*7GJn~90t zo{+@?Qq8dW05Z$cvoH*9)Sg~1Q@w#FAK$~)*EuCp{T)t;R9849QoWs1BGoT)N~HSV zI3-g3OHPSY@8pz7b;5I5B)Au+M1qHi{N}yIXpQ(+?1Zrh*4wmo)hlaTUu=!BK2k1D zG!wWHPokB8%;yB8OlcVT-bc`9u= zJ`{WokZKm5R{@zvi7#tA;;;NRuZnQakF`xQD2X$VNDfLb5QEY)tHaJVyn(E?@QGd5 zvdf_O&9w_z#14aEnvcU&AMzb9~Bq%O{+- ze7bqdXLDO?1wVXOR@RqS@IzWS5`N?=i)9rnDi%|$qgX<*dSc^D1{LdyB4(^t8+|F}SvUmY}327T3d^hL6qBe_uOjxO4c zY$crXQ&Qh+w^Oc1zm*h-U1~hmWoHk(N(}6G=S{5oeZ7)*{HBCib-P(0r6+NJqA#OT zbXr5D*Xnj6DY2;=KX7<{aTm_k>mT76!#fA=S^3@h{y~q*?T@85Da39Ap}B*2D!re4 zYS%ij!Jypv6b}vVg6(?Cz~Vj&JaYMKJCJ!J^tH_Q<%Ox8gptho5<)&#uMKFSh8GQK`InO+rfHEh^REP&0LEI4awWd%G(yh(Na(H3)R5~OVhxoBIO$7*clHr?b(xO*Stw?iNF*Yj?8Cjj z>dS^f^B8D3p>1psTi4uAeM)b03mOAlNi+P5waJx zEz0hf4*_;j>Z$%Cnp`BT<^!Yp@d)ymJ0Bc%h1LLV$F(}OCkBVkBL|E;cxQ#yQ}vGJ z!5*4X?I=K&vU{_|{6l{{IXYfkB>{#ZL&QbK>i~X5@hX6~yjH@i${HaRWSt;NgkJT2 z0+~W8AvX-L?|EVZpE<+@+H};QJ{dz)AyiMNkPyn% z)O+J_4mbsuPZCcxY&`KAWC7U`P_k9$!+tZ(SA8v=d?@T;`$l4RI(C6W6LIC=&LFdd zDIOsP)jf+`lDeChl2}O0B6`&{nc#WLd(}@Rc>Qt8Je_Qn9pWk*U{4;2QNPM1Z%e(l zJmM<#PRt}OQV;!Ez%q-vs$R4Cs_@uR&jKZ$R0op^F=d}ygDIOfhv0$wL#dmZHIH18 z7)B7gb2jQ`72x`oEch>clNaKf77@HjxDUsOJ)y6Fdb5;fU-ifmazY4I3`sW_oO*s4 z!CQ%s?qG>sz_QjD)Qe^0X`#lCCAGL5_k{3`>F23tts;*GSwJ>ityzOx%&8(UKuiY~ zf2vcrTgiDL9bD_(emE`7m5!h>!42wjRgU1q7=+5H8gfHA#>37N_UDm6-~#ZOhmLP- zAeW^S=hf^KXu;#M))n55z+I}VCo|>3tbB$9N($W@W|syh!Q@n_d*k!pml{ntUBE#LLjgA*pc`Nc#^A_P6yDVxhu);uVPXl06c@NW7t@hUK1Zh@w7lvxs`Am-g^|B%j$0Nn^?(s*~5%z)xDk65>`0H zPi;BNcL}GKeMr8MRPi~2#bV(UqiX#F`CoxS@kL@(U;HDvDpl}?{@Sj5(N6Kg$OxZb z)w`%0FY}$xoi4sgIt0b=@`cqGu>AP+ir!t_e4Y5vV*WJ9m)`~QUBtUBpNdCYez3a|b-}$qlPLxKJ%We#|MT&H zMis(4HxkD)3F^(6fH0uKA3tq~wwM z=?%Q}$z%1co1}%h$houjtLe8%lRN29&woXZQ_`t!xli)kVc1CO54XQ1zb1X^mTpo` zNVodMcf{X~Jb@k!%^*E$nwIwZlL0j)h#q$%z3PJ~8lxp$YS2i!N8`y?zfsIzqxCzu zJ(>=S<&)AVR=H6ua-)d7QLI6u7+s@S;6_|uuUJ}49>Ka;Iu4s($I>YV?$O#Y60)Y! zVc>d`#CTapj~+6_zWkjdp)?%@g>iH=T09>|$B@TxAPz%)4EN(`u3mgYAn^TI+*EVp z>2L{X!IFZ&j(9o}6;H*}Ol2z$SB+wMn56>ph9BkSX=rnBfiF7_TOA~1UHt#0V zJZzpM3R$`Z`w8Xj(xrJIq^Y>zl%Noe(*}7lhMl@*|a9n zG~_NO(W%mIJoF@@GkjvP2b(;ha11p_!NeapA3|0X5@jE;jFvqZxYfz^{eVTW;0~3;` zof1C@Y=oKFICU+lG*SXDNDX|Q`JvB;sdOSX@o97d>4Y_D*lj25OQZRE@e#oWhtlX! zIJk)F;r2ut$ZAgO`9sUA)xHEms~No0X$r^TVVKsEJgCOk?)3W*>t8}eA;=y z=XeZ%{P@_@u^S>Mezn`4vdEUDYebtn3Z-xKWfEqn}!g-C)c9>mi*+uiWy z6gms-A55Wn(%t}X=3rkw{&+8Mno3IO;E=s59^fs0eAJDS{>O6Ot|H;vB0X45ng z$0yV10^}m5(|Op`O&4IhY&>#s%XGt>91-|0bLdRuzRJNv=?Mhn;y&tu1-S_Jz^`-Z zDr{V5&?S0t4Q0R|XV9r|=Ovx5eFA#mqZzamm4@fhVuMhcJ_>yJIk!6x_f!vD%cBdV z^-Oqk7Up<$0Ua$We%O#t{iSHO6t(5!{uNHuH|P6OcI^Bm*&g#>2Jr{kr@ zE(x^F$MaiEcbfzT=FmB9Pb2_Avb@Y6-qL!u|0s0GdMETS_c z{(%#ZhBb>F+UF$Nyckcx0cc;0c|QOiOYkHcfZ`=MA0qyvP?Sr9!4!dK3s&GmQTSj9 zu8$D@NyfFb2+xE8*j0qv3dKcqIW`4L@zx!H9ZPAkRQ{(_b}z=`ycY_J=^8z+daU2b zIIb3>R{=bfK;|->aREG*z^-L94lO@fMwd&xOM0BV9FMDBur0@nwhPWJ$CIdQX9-?l zU656RXIB@zQbNaLbGigmq6_*Zms(0Q4dNF8ztM1J9HwFSC_H4X@}dxep`+pSSQM0h zD2rhacu-39=%TBXriY794wg|vhO9|mliXO>x~0{6Efxp#eH^c2dZ4h3hT;rV$;Kv| zcX`vReq2U>yA~^(r!L?Yn2cYyogN zjs}Wy#})=&@iaoqYa!V3keaZ{Dej<=*lg;cA=vz)gPO@>^>hcl{EU{DGLXGm6Dh8J zTQEyn6`~V0JRGjj#7lWYg=Qq)^+{IE2#4D4st_#o;W?`&8YR1}now*`Sv6*C?pifr zq<`l+O;`wbZ}ZH-AKGT{AAu8ASK!aJ0yM(IBpTvycXv~WLE}1aHJJ{Tv>*7S&;Sgmpi&coO?9QlgpI9I6Ds}G zta4Co<(i;$5m&%qu;N$WlKKt!1GkJVa8OoXgFk>*l&q*Mt6GOrh+NOdtY)p(+=~+B zi!B^p9Y-Uzyx4*GX3a1xUaDF&!D$?@`EyTs{I_VFk&@cFvbuWgM2JKHPSMj98^o{Z z)(&V#Yjkq6MPtNb{z;2waWme_1rgc^Y^o!)G2;HVg|npN{?HPp^+NcJ6At9?&Pc6J zYJ~}{0~4sH^by3cdeO96g(uE<8^qfUQ}FI5!8$=hcBpokG!KEZk4ONthiP3U5XAw^ z&QAC~QX4EWBP;xqQTR-c(neuF%cHcTjJ(LO`S9`2vi}=_%!*EEi_(UpR(q6oBsRze z@*>6F!7$eTk71!0AUF5>k-7Y@0qi>4tPMuXVl(ee-E7vbbk*}Gi7g09yNQunzg01% zFNNv(bImr4ZSD%9);{RS*9K}i1P%EZL7y5sQ+vEh%gYVOR<)5%3UieIdxNzbEGM*) zqN1>cKu0ez8N_qP77DMrkWk#P+*Wc%r_GrYTC=nXg|i;Sq{S5%o&tE4V(_S{F0Zg# zngBo0m~iT_usNv>67`CE8UD|NbQ}d_8Xf^c7~=@zZW+#(Mb#Q;H-*FDBpjMcv~G@A z%z$#UgA3tYuqzLM2O0uDr_?cq03ahsfan`X8Gs*J#Rpiu?8gKhUx zXBz=`Q^-i2c;`X?OFEO5-->YQC7oF+ghN?6iJ*7@KDAXB70abIuc_6~R8>}&mCv)M zzd$`4XxR$-pwZU9hjvaG6zBwRTkn|HqFW&~MX41q#^H0Vi;uT$Z^_ z7b*kEdPNr{0|^2AN@hnRU(xxa=abuXk@(oXy-l~W(7>;-THA0q96|%|7WP%wf2Nx` zKa9gRpYn>z3f%O#5GAYY*Oue-34ADQE~3Nmo%|qH&(g)kR%?U9+sRO{#OZz1{j2q7 zXX9m5t?JFW%a5}{)N delta 9161 zcma(%d0dp&wca}$`!2E!J1WXH2;zclDuN)2Ta1V-3N9luge9q21e0isP7BFJ&r4G) z)+RQwu3zHkf{Iye662briM2*;e@&XiR1pM3G(Gp;88Gkt^Zw!N=bn4cx!X4!c{A|H zn}G-ZHl{)MRDDIwx>l>M&SI&oZMC`y7{9inuB^PQuB_GSR9nAn?dsav)t0JOt9Hfe z$_fN!^>wRSt(s?Q(ZXR!*JixBs=VSCtE*O`YzWnwEz5@dm)F#mRM%HldbV2imSyIO z<#iZ>P-f<_kqtd`*f6UDA1tn>wyxEDkJFAqE*XzF!^Q4h^7@4lA4bv-gpNHr$?M}h zCpn_zJmeflkp;=1-FE!o{dPh>|j?sA3!Qy|G+mKp6&2H28l|G@iPKWN%5#Q8wwA;BV;_X&4$I*?+S`Xx7qOd^&^7f5pFhoYdR_@9_41k?>i;MBi(Fx z+k8RD?q&)yr$drFP&7>ZTnKrvn+-Fvz7kZIpxT442#QY*2fTIVEE{RcB|WU#6hQ8= zEv5hikF&Woip?G89o!-^h?LbmE-rp$MO9+FXpic3tu5w0cFhz_9KM_L5A#lpW#5d8_3^bklGUxRTU}YpO2;i@UySo&eF@Pd zlNBe9XWu8dvG>M#IkuR2;5?q9#9}sYoHsj`IF0nOM~Qnd>bH|Fvz?P7*pBf_*^bE( zEF`&(84(^%E@r_goUdo86Be;R)KMptF_s*`QYMx#p9vA{#KdCOiZE@`atyg`ayV=J zQ3vZ|i8uTEqzv}ki4iP)aw!`a9|6}Ih>?9gIfHbeUIaAmBY|vbN(S>oosN`x*^Z68 zLHIf|zs51qj&E{Iw4=f?(T*J)6YY48W1=1Z#xc>3-*QZ}V>ib{I}+Y+q6ytNCYmr% z^dKKCO6!k*MYoPeX1!HgXIWm``a)}zxkouSC4#{E)5Mdt9r9Lu(w-HRm(rGg$cB|? z-m+oKrV+|@z2JrI#GpK%^+*a}-z&txcBi{4ZP^|aoMMTEh2&Pil~@v~{3*AJaLkkK zNOw@;XQ5k8LCIO-p!DZ=+Q^32k<=F4v+2ffcTl|M*@YdWP-tB6<71|*Ki^F;J#o{f zKj)f*GWJO$g{lS;1dWL#05W39B&>H5`@S#-s&~j4vL}|j*C4hwLq%0x&FYHU)}^gc zv(0=v+|)W-l!rrJ5)OGmIOOHvkQakPUJ4F*Avok^;E-o|Q)>m^IG0z}msju&RtzCV zkkyYA^%kWTRTc#nwH0L*7421y7dcZNwio-V_!eZCFsFx;gW25D9Pn6ZE7yrq3*nTP zlD6yvi9zG*Sen4{UYj6tg%g!38LPY{1_zo^rP53#xvQ~KA}n%6kg~rDQ_5WmM=Y@1 zS2yjihZ6Cod))^pnrN+S0%RQ9`PoEkR!TJR1#=d{rQ=#kL zwYmEES*#`>hlOqpghvU~MY;X_XEx#I>mN!?c2-B72MhhRR(aI;cbmek7i}XeoiBIW zc)gqK!SNJeuf%LQYjffF>$?&YK@}xY z7dW$%cq&=VCv5!QO@T`MD_`2EeKjL^U~v6g;;2kvmu+rBx7uQ2w_Y5fq{3G=bxx(d z_O~HpIPe{&G3WKa+6+zGL$DP4u+(y28o{pa$bq!OwsL2;;Zj1%i?yX2l=C|=C{8Kc zclFt@-EZJp6&kzW7!IWqs8%_-8?*X2%Nde0a894U)h_wKv!T4T=bOjWu1%oMY|lSE zmCJjNNw)I){r;4}fCab7FqVBN2TGmMOWUE1gkxUHgm*e@nB!q=KF*W9=zpZgMv86X z&u5Ixa#XM69=j^p%+e7dIHm2_dT8gb6rR*j>DAJKTMqYy6)`AH@7X@?~1ts$8b|-+cIXf(P zL@8+#`6eyja>YY>{qN5~l42WnIrVf>567>y+GrYX;@Hej{b-K?|Ol z5p2xEoL7%XKU*Kxkynp+iB^N%?pWfKpH#tVb0=XCa$M^K2l|N)HaikeHH(lBeuI&A{Jk^&BL*`M?a$M`6qT9dPJx0MP zCt`p+4e?SHXYv82;RAoQ#g#nb8Y%{)p|+7!9O;3SU-TxFvmyT_G>#1*!O+~Ubyj_R z$#r3;^CRwq3MwiPPN^7~i$IEliI0%C>?8(tOQ;-0Z40;g%~Kf^~l*mE;Rk+&c`aa|SstbsLtDXo$}s zdeu0Ud?m-%PbbcD{D>K3qZ|-xY=GUlBuf1*hrA*6+Hwgl8XocSS;Rr=p+6It@~NZh zHiy@Q$Bud?C~>7aNH4&KeRdT#?8LdG{pV6AHDf-xD0vt`^tL&un=u=!Te9%~@YlN` zfBhnIS?b$o>LGBKbfMkeTy$ToDpo-uD@^oPGraEQp<3_o!n8D3+JnaS zHmJ{5*@F{v5Gp6C$rb4scR5$sn@fCw3&3MOI=)gvE=VcnHDXYp1(yq2M>raaYgAWH zX34@Ve}?!<4BZ>#^sx6-?7g3)lYMX5hwxZ{75d8p>Z|s@K z?34C5(_0e5Bkx}VZZQCgM2L2u+&-^*qggZgv74CLH0{iTsWX^LDSH1>H${^ zsf&7kH(y75u9!w|VIWuPjXlIf*@2lBXxN9D`(cjEoMAulAV_%uy7f_05AdbLNnGfz zu=FtT2U8Iiz;uYOMHHw3?~q6;u!6$`j~uQnN+y)AE~I{N>n&VQXOHl9;}*M)+4NN0 zGg2u#`hx`@RFSab6v<)Dw<1;N4pK`H^HiHp^F_$9qW8&vB(~&3(jzc-=Vb@g{1@_X zLBW!9BtU)PV{%EV;adcA%S}-rG5$UYfa{m^4(hrKeC?x@we59KOD>V`CAGK40al*H z8^EU*_0DR;XT*aR@pnX?eCFplh$mhi6_1I$5b>7Ca}-aBJfnC?=hdr$m-o)c(PA-r=_@luqVJ3RkaVieC) zd_jwNTf`$}h~nOdDE?|0qWJS=h~n>Y4AzF-rQ>4Yx?HGkFM~_L6rg91SPN(W~nyS?5O z3dT``6ij){A(uXCbRTlsHjbvFn;YZkGz5`}G(#HY;Cotn8+;x*8)PQ?VP7I9pdWrr z#Ju%GbP}D8pgM_WAUK#rGo_0{p0pUq_&0F@(|Bq?;l%M`v|mVYWIWA8X8(9Guhe8L z#sjEMra1_XCeu{C_yAA~CzI(Y@ls*)kGY&5B5x?d3y1Y!3qZ^(&4aw5!J)jQ>ZViKB?!gF1J=& z34~TNxTnx0&WAg}S|=Bo6CrvPPE5t&_fMiGbbMnHorAzMnNBwd@#|5tD@WZSQLc}> z{5>yFoEYxwkMxd%!+}$Za8Z~-vt@|8dBga@sT3M18E=DIDvf*f!eB-FAZ`jRz;PR< zU~>E5^b~B%K5$H>GZD;Br3KQkU8ZVV8ukDA{7wNL)_%wzBNg9+(s&#)JB=nt#)o!= z0R94VN3R`e*w=laPsipFGd({TrY}42ogNy~sRJrBrHddwOULE<5S%iwf4end^{3WNPvqj#C0&$-LP{S%}4v&(`c@=H^9zp9LvKS59;;P zse|M?K|&Ty7VjE3H=R18$(8AJ9)g%GIv2tEED^^iS#%*%p)=@w1a&h6vqJ)xY+Nth zFgIHS{<~~C3#sq2aYwofJ~_CE24Ga&bWoz~x-JP+HG|o%z_{mf3WycmhC89`%-@Ia1V? zhl^J@nI{fpff z{D$u%xhn^`(!jG2otx)BmL> z9cka^2B4#W&XZ0XY{qlx6v=pTkby55yvh$5ot{hQOPyayo%ng!kpoaQkCq@fKM!~J z0SKB;7a^#hPfMgSz$?Yn2mUkPK9Ao@M*adiNos7DjJ5^1d5d(nNyeiEbh2JtJ9f(f z-&sgU!*+k_XWuY{@a;k245__WGOo_W&vEU6)Nz|X^@5WD)Y(3l`!MGT+*j|z|2#oU zrHvwo^jbtGNXAj=1oyv4_|+onBDw8@Toan#SVU(^{(FP`NLclxUHe1HZGIBB!28hg zB=-G%a9NDo*nKEkjQJ4pABMsl8UV&n+*t5RJ{*O&7h`>d@N*g0l0w`N?!)#%TvjM9 zq{RqkFTsuBK5Sh=i=^^DrLuDo?&V!DyNIsR^9zvqM;XVZBJ?U4cO)ZiDP~+Sx+G)! zQW}GnpDd-tlHVgePA$g0stc^ec*^#|hsC%p^=>P{)2kOUN^s-qg_ld{Bm^f*uqAro zp`<31(lmqk9l>iXoJz(v>>GnStXZBCLNI(RoE(pW^7rMv*adEvQa!rpDWxeP;v0l% z%&=jr5?3WQ)wOPDHD8X#3H=nqFEIm9P)36=162}OC3u^IF7=}_`lr=+t$FG!E-`)) zvih?PS^f^K;wEejMAN-xDZ2q#a+i53~sSBRLd_yu;C6h z;uWW;lZGK!-$?@z{I-)ukbCOMPI}=PEx*S==1NVNIP!{h)1)#s@CqXKoo!$4ZUAv<#9W z{P^uG8QyRykq(!*7kDI59}H-Ar6v@CrBY)=V6D^yNk7#q>{wg5#xF(0Bb@e!i5QzW=MNe1Mh+oak51<3B(aF^ojX!yy-fPi3 z*^Gzs>`-ke0!yeiN*vzS5SDPv8(M<3Zpc10$oA#zZDCrS)Cv|_`zBLY>BEoV)kV{4 z7M?iaZV(rGY{8qK1n2|~nL*kS(mW7O-z7fK5v+BTjBw7t?(Bn~!n6UB=g$g0_ZL1> z!nNTzPI0()j6c6&SUvds^KJhhAiJUu+QPLVsMQg!9gP4fUw%We4KRXr{9{BA2FT65 zer7KJ`v8Z|jL-(4Wl;nlO>KzKE_c-PC!5s|O8baEH9u6brOyZJ`OD2Zf;IH`QFAAB z=4pMkoCS?}7(u5RJxhD6O3SYskg00J1~KeW{!a$xZZI9!hKU=6H4r)<5Til7b*w?~ ziUSG41bN#S+62LA7h=?6#f2vyeoHa9R9VU^Y?emAFE&O@9bRnOwSMBh z;#^i)X%mM7$-*r_2&3#_+$_Ue!S^D62SMfboFUK)LXSr5Hrv7+B`|TNV{8>lA%_K5fCmwn5@I{?b z%TGl(|DrBJDuh5;3JImS0Y0@+7aq-}R=4SvXR0bKW##j2?Ju}4c5c}+{4WtSIJr@0 z#N+wuMqPlX&~V5vt6IIHg8xg$0O&XA!hHqX)y|INKDjJylP*XGlJT-GTm}*d_}R?H z4SQMVjh@>#>B8{o`}HQ>@&W@t!fLG};b0*3!Bf~%UHheO)`DQpwtAFTR94`k$3m2> ztY2M@=@b0n&`?N6ko)TGXgy022V1Qb4sIdC!4#wSQ1`CXpPqw3#7GNxiHO(sPD20Ln_JQ3|Us>3t;jn@f6si5(o|T|i!_un}9w rqT2a&mYTAa6>;^|G81Be_KSL>FURVw4l8l-#qs~UscWsTUl#XYb`p@Y diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index 27029214c1540e6a18aa0773b4e6765e608c1318..e6ed6873e0f369b373217fa4fe853b5b9ace0a13 100644 GIT binary patch delta 173 zcmaFb$M&F)tzinI5~JBxWkxdI)MWCwfftHL17zL3e^>Xsl)ioy)gX!oDrVUJ5+@{7RdWMEZrk0i#lN0o9HhXIEup$d?PSD@Sfh;gt(;^Us-)iB8 L%-?*?B0m@a#Htwn delta 96 zcmX>)gX!oDrVUJ5+$KgwdKSiJX69yQlN0o9HhXIEup$d?PSD@Sfh;gt(;^Us-)iB8 L%-?*?B0m@a$BG#Z diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 19e643a38..5b27d3588 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"DRS Download Tools": [[10, null]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Gen3 Auth Helper": [[0, null]], "Gen3 File Class": [[1, null]], "Gen3 Index Class": [[3, null]], "Gen3 Jobs Class": [[4, null]], "Gen3 Metadata Class": [[5, null]], "Gen3 Object Class": [[6, null]], "Gen3 Query Class": [[7, null]], "Gen3 Submission Class": [[8, null]], "Gen3 Tools": [[9, null]], "Gen3 Workspace Storage": [[13, null]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Indexing Tools": [[11, null]], "Indices and tables": [[2, "indices-and-tables"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Metadata Tools": [[12, null]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, null]]}, "docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "indexentries": {"_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager", false]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods", false]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS", false]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create", false]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases", false]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record", false]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias", false]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases", false]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest", false]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get", false]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases", false]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record", false]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum", false]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page", false]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params", false]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest", false]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls", false]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd", false]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait", false]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update", false]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases", false]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record", false]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest", false]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider", false]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ", false]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create", false]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens", false]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children", false]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID", false]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url", false]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy", false]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create", false]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases", false]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank", false]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path", false]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job", false]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version", false]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list", false]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program", false]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project", false]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record", false]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time", false]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl", false]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR", false]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete", false]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias", false]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases", false]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids", false]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file", false]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations", false]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path", false]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node", false]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes", false]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object", false]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program", false]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project", false]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record", false]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record", false]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records", false]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download", false]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download", false]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download", false]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest", false]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single", false]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url", false]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable", false]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager", false]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus", false]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time", false]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint", false]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node", false]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record", false]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name", false]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name", false]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size", false]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size", false]], "gen3.tools": [[9, "module-gen3.tools", false]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download", false]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest", false]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest", false]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest", false]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest", false]], "gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth", false]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File", false]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index", false]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs", false]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata", false]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object", false]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query", false]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission", false]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage", false]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get", false]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get", false]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token", false]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts", false]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases", false]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records", false]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all", false]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node", false]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token", false]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema", false]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix", false]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths", false]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version", false]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output", false]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url", false]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs", false]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary", false]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest", false]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects", false]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record", false]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc", false]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records", false]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page", false]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats", false]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status", false]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls", false]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids", false]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version", false]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version", false]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version", false]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions", false]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params", false]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query", false]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID", false]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT", false]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT", false]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname", false]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest", false]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE", false]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy", false]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy", false]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy", false]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest", false]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object", false]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest", false]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs", false]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load", false]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest", false]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls", false]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path", false]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest", false]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS", false]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5", false]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum", false]], "module": [[9, "module-gen3.tools", false], [10, "module-gen3.tools.download.drs_download", false], [11, "module-gen3.tools.indexing.download_manifest", false], [11, "module-gen3.tools.indexing.index_manifest", false], [11, "module-gen3.tools.indexing.verify_manifest", false], [12, "module-gen3.tools.metadata.ingest_manifest", false]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id", false]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id", false]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type", false]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project", false]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint", false]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID", false]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query", false]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query", false]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query", false]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls", false]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download", false]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token", false]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects", false]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm", false]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path", false]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE", false]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time", false]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status", false]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file", false]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record", false]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl", false]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER", false]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update", false]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases", false]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank", false]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record", false]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time", false]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload", false]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file", false]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid", false]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url", false]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS", false]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access", false]]}, "objects": {"gen3": [[9, 3, 0, "-", "tools"]], "gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "terms": {"": [1, 4, 8, 10, 11], "0": [5, 7, 8, 11], "0084667": [], "013444": [], "0248635": [], "0791543": [], "0a80fada010c": 11, "0a80fada096c": 11, "0a80fada097c": 11, "0a80fada098c": 11, "0a80fada099c": 11, "1": [4, 5, 8, 10, 11], "10": [5, 7, 11], "100": [8, 11], "11e9": 11, "120715": [], "1329331": [], "1468077": [], "165499": [], "1724170637": [], "1726237606": [], "1726237962": [], "1728059996": [], "1728059997": [], "1728061593": [], "1728061594": [], "1729113765": [], "1729629354": [], "1729629627": [], "1732401632": [], "1732401820": [], "1732401821": [], "1733254905": [], "1733263055": [], "1733431971": [], "1733932163": [], "1734031012": [], "1734031066": [], "1734031067": [], "1734474422": [11, 12], "2": [5, 11], "2023835": [], "2310405": [], "24": [11, 12], "244287": [], "2511687": [], "255e396f": 11, "256328": [], "284675": [], "3": [5, 11], "30": 8, "313695": [], "33": 5, "333": 5, "3410327": [], "343434344": 11, "363455714": 11, "3686683": [], "3697538": [], "3759608": [], "4": [5, 11], "4401503": [], "450c": 11, "4714": 8, "473d83400bc1bc9dc635e334fadd433c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "5": [0, 5], "50": 7, "5117643": [], "518752": [], "543434443": 11, "5833347": [], "6129959": [], "6138082": [], "6146133": 11, "653854": [], "6f90": 8, "7d3d8d2083b4": 11, "8366766": 12, "8420": 8, "848236": [], "8493605": [], "8535511": [], "8712757": [], "9082778": [], "93d9af72": 11, "998105": [], "9a07": 11, "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "AND": 5, "Be": 1, "But": 5, "By": 11, "For": [1, 5, 6, 7, 8, 9, 11], "IF": 11, "If": [0, 1, 7, 11, 12], "In": 10, "It": 10, "NOT": 12, "OR": 5, "One": 7, "Such": 9, "THE": 11, "That": 3, "The": [0, 1, 2, 3, 5, 8, 10, 11], "There": 11, "These": 9, "To": 11, "Will": [4, 7, 10], "_get_acl_from_row": 11, "_get_authz_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_for_row": 12, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "_guid_typ": 12, "_manag": [2, 9, 10], "_query_for_associated_indexd_record_guid": 12, "_ssl": [3, 4, 5], "a5c6": 11, "ab167e49d25b488939b1ede42752458b": 3, "about": [2, 3], "abov": 11, "access": [0, 1, 3, 7, 10], "access_method": [2, 9, 10], "access_token": 0, "accesstoken": 0, "acl": [2, 3, 9, 11], "across": 11, "act": 0, "action": [9, 11], "actual": 11, "ad": 3, "add": [3, 5], "addit": [3, 5, 10, 11], "admin": [5, 11], "admin_endpoint_suffix": 5, "after": 10, "against": [3, 7, 8, 11, 12], "algorithm": 3, "alia": [3, 5], "alias": 5, "aliv": 7, "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "allow": [0, 6, 8, 10, 11, 12], "allowed_data_upload_bucket": 1, "along": 2, "alreadi": 9, "also": 1, "altern": [5, 11], "alwai": 5, "ammount": 12, "amount": [1, 9], "an": [0, 3, 6, 8, 10, 11], "ani": [0, 5, 10, 11], "anoth": 5, "api": [0, 5, 8, 11], "api_kei": 11, "appli": 7, "appropri": 13, "ar": [5, 7, 8, 9, 10, 11], "arbitrari": 0, "argument": [0, 13], "arrai": 8, "asc": 7, "assign": 9, "assist": 10, "associ": [3, 5], "assum": 11, "async": [3, 4, 5, 9, 11, 12], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_create_record": [2, 3], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_download_object_manifest": [2, 9, 11], "async_get": [2, 5], "async_get_alias": [2, 5], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_url": [2, 3], "async_query_urls_from_indexd": [2, 9, 12], "async_run_job_and_wait": [2, 4], "async_upd": [2, 5], "async_update_alias": [2, 5], "async_update_record": [2, 3], "async_verify_object_manifest": [2, 9, 11], "asynchron": [3, 4, 5], "asyncio": [11, 12], "asyncron": 5, "attach": [3, 5], "attempt": 11, "attribut": [10, 11], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "authbas": 0, "authent": 0, "author": 1, "authz": [0, 1, 2, 3, 9, 10, 11], "auto": [0, 2], "automat": 0, "avail": [1, 2, 10, 11], "az": 1, "b": [5, 11], "b0f1": 11, "bar": 10, "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "baseid": 3, "basic": [3, 11, 12], "batch_creat": [2, 5], "batch_siz": 8, "behalf": 0, "behavior": 11, "being": 3, "belong": 8, "below": 11, "blank": 3, "blob": [5, 7], "bodi": 3, "bool": [4, 5, 8, 10, 11, 12], "boolean": 3, "bownload": 10, "broad": 9, "broken": 9, "bucket": [1, 6], "bundl": 10, "byte": 10, "c": [5, 11], "cach": 10, "cache_hosts_wts_token": [2, 9, 10], "call": [10, 13], "can": [0, 3, 4, 8, 11, 12], "capabl": 9, "case": [0, 10], "categori": 9, "ccle": 8, "ccle_one_record": 8, "ccle_sample_nod": 8, "cdi": 7, "chang": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "child": 10, "children": [2, 9, 10], "chunk_siz": 8, "class": [0, 2, 10, 11, 13], "cli": 10, "client": [0, 3], "client_credenti": 0, "client_id": 0, "client_scop": 0, "client_secret": 0, "code": [2, 8], "column": [11, 12], "column_to_use_as_guid": [2, 9, 12], "columna": 11, "columnb": 11, "columnc": 11, "com": 7, "comma": 11, "command": [10, 11], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "commons_url": [2, 9, 10, 11, 12], "complet": [4, 11], "complex": 7, "concat": 11, "concurr": [11, 12], "configur": 1, "connect": 12, "consist": 3, "constructor": 0, "contain": [0, 2, 5, 8, 9, 10, 11, 12], "content": [3, 13], "content_created_d": 3, "content_updated_d": 3, "continu": 10, "control": 3, "conveni": 10, "copi": [2, 13], "coroutin": 11, "correspond": 3, "count": 3, "crdc": 0, "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_blank": [2, 3], "create_index_key_path": [2, 5], "create_job": [2, 4], "create_new_vers": [2, 3], "create_object_list": [2, 9, 10], "create_program": [2, 8], "create_project": [2, 8], "create_record": [2, 3], "created_tim": [2, 9, 10], "creation": [3, 11], "cred": 3, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "csv": [8, 11, 12], "curl": [0, 2], "current": [6, 8, 10], "current_dir": [2, 9, 11], "custom": 11, "d": 5, "d70b41b9": 8, "data": [0, 1, 3, 5, 7, 8, 10, 11], "data_spreadsheet": 8, "data_typ": 7, "data_upload_bucket": 1, "dataa": 11, "datab": 11, "databas": 5, "datacommon": 0, "datafil": 10, "datamanag": 10, "date": 3, "datetim": [1, 3, 10], "dbgap": 12, "dcf": 8, "def": 11, "default": [0, 1, 3, 7, 8, 11, 12], "defin": [5, 8, 10], "delai": 4, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_all_guid": [2, 9, 11], "delete_fil": [1, 2], "delete_file_loc": [1, 2, 6], "delete_index_key_path": [2, 5], "delete_nod": [2, 8], "delete_object": [2, 6], "delete_program": [2, 8], "delete_project": [2, 8], "delete_record": [2, 3, 8], "delete_unpacked_packag": 10, "delimet": [11, 12], "delimit": 11, "demograph": 8, "deprec": 1, "descript": [3, 5], "desir": 11, "dest_path": 13, "dest_urlstr": 13, "dest_w": 13, "dest_wskei": 13, "detail": [2, 7, 10], "determin": [10, 11, 12], "dev": 11, "dict": [3, 4, 5, 10, 11, 12], "dictionari": [3, 4, 5, 7, 8], "did": 3, "differ": 5, "direct": 0, "directori": [10, 11], "disabl": 10, "discoveri": 10, "disk": 13, "dispatch": 4, "dist_resolut": 3, "distribut": 3, "doc": [7, 10], "docstr": 2, "document": [1, 3], "doe": [0, 12], "domain": [11, 12], "done": 4, "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "download_files_in_drs_manifest": [2, 9, 10], "download_list": 10, "download_manifest": 11, "download_singl": [1, 2], "download_url": [2, 13], "downloadmanag": [2, 9, 10], "downloadstatu": [2, 9, 10], "dr": [2, 9], "drs_download": 10, "drs_hostnam": 10, "drsdownload": 10, "drsobjecttyp": 10, "e": [5, 10], "e043ab8b77b9": 8, "each": [3, 8, 10, 11], "effici": 9, "eg": 3, "either": 8, "elasticsearch": 7, "els": [0, 12], "elsewher": 12, "empti": [8, 11], "enabl": 11, "end": [5, 10], "end_tim": [2, 9, 10], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 10, 13], "entir": 8, "entri": [3, 11], "env": 0, "environ": 0, "equal": 7, "error": [10, 11, 12], "error_nam": 11, "etc": 8, "even": 11, "everi": [9, 11], "everyth": 11, "ex": [0, 11, 12], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "exclud": 3, "execut": [7, 8, 11], "exist": [1, 3, 5, 6, 9, 12], "expect": [5, 9, 11], "experi": 8, "expir": [0, 1], "expires_in": 1, "export": [8, 10], "export_nod": [2, 8], "export_record": [2, 8], "f1f8": 11, "factori": 10, "fail": [8, 10], "fals": [3, 5, 6, 10, 11], "featur": [1, 6], "fenc": [0, 1], "fetch": 0, "field": [3, 5, 7, 11, 12], "fieldnam": 11, "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "file_nam": [1, 2, 3, 9, 10, 11], "file_s": [2, 9, 10, 11], "file_st": 3, "fileformat": 8, "filenam": [0, 8, 10, 11, 12], "files": 10, "fill": 12, "filter": [5, 7], "filter_object": 7, "first": [7, 8], "flag": 11, "folder": 11, "follow": [0, 11], "forc": 11, "force_metadata_columns_even_if_empti": 11, "form": 13, "format": [3, 5, 8, 11], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "func_to_parse_row": [11, 12], "function": [2, 3, 4, 5, 9, 10, 11, 12], "g": 10, "gen3": [10, 11, 12], "gen3_api_kei": 0, "gen3_oidc_client_creds_secret": 0, "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "gen3fil": [1, 2], "gen3index": [2, 3], "gen3job": [2, 4, 10], "gen3metadata": [2, 5], "gen3object": [2, 6], "gen3queri": [2, 7], "gen3submiss": [2, 8], "gen3wsstorag": [2, 13], "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "get_access_token": [0, 2], "get_access_token_from_wt": [0, 2], "get_alias": [2, 5], "get_all_record": [2, 3], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_fresh_token": [2, 9, 10], "get_graphql_schema": [2, 8], "get_guid_from_fil": 12, "get_guids_prefix": [2, 3], "get_index_key_path": [2, 5], "get_latest_vers": [2, 3], "get_output": [2, 4], "get_presigned_url": [1, 2], "get_program": [2, 8], "get_project": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_statu": [2, 4], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "giangb": 11, "github": [2, 7], "give": 1, "given": [0, 3, 4, 5, 8, 10, 12, 13], "global": [4, 5], "good": 3, "grant": 0, "graph": 8, "graphql": [7, 8], "graphql_queri": [2, 7], "group": 3, "guid": [1, 2, 3, 5, 6, 9, 11, 12], "guid_exampl": 11, "guid_for_row": 12, "guid_from_fil": 12, "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "guppi": 7, "ha": [0, 11], "handl": [3, 10], "hardcod": 0, "has_vers": 3, "hash": [3, 11], "hash_typ": 3, "have": [5, 11], "header": 11, "healthi": [3, 4, 5], "help": 11, "helper": 2, "hit": 11, "host": 10, "hostnam": [2, 9, 10], "how": [8, 11], "howto": 10, "http": [0, 7, 11, 12], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "id": [0, 1, 3, 5, 10, 11], "idea": 3, "identifi": [3, 5, 9, 11], "idp": 0, "illustr": 11, "immut": 3, "implement": 0, "implic": 11, "import": 11, "includ": [0, 3], "indent": 10, "index": [0, 2, 5, 9], "index_manifest": 11, "index_object_manifest": [2, 9, 11], "indexd": [1, 3, 6, 10, 11, 12], "indexd_field": [11, 12], "indexd_record_page_s": [2, 9, 11], "indexed_file_object_guid": 12, "indic": [0, 11], "infil": 10, "info": [3, 11], "inform": [2, 3, 10], "ingest": [2, 9], "ingest_manifest": 12, "initi": [0, 10], "input": [4, 10, 11], "input_manifest": 11, "instal": [0, 2, 11], "instanc": [1, 3, 6, 7, 8, 9, 10], "instead": [1, 7, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "integ": [1, 3, 8], "intend": 0, "interact": [1, 3, 4, 5, 6, 8, 13], "interest": 10, "interpret": 0, "introspect": 8, "involv": 9, "is_healthi": [2, 3, 4, 5], "is_indexed_file_object": 12, "isn": 1, "issu": 0, "its": [1, 3], "job": 2, "job_id": 4, "job_input": 4, "job_nam": 4, "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "just": [5, 11, 12], "jwt": 0, "kei": [0, 3, 5, 13], "know": 11, "known": 10, "kwarg": [3, 4, 5], "l": [2, 13], "larg": 9, "last": 10, "latest": 3, "least": 3, "level": 6, "librari": 11, "like": [3, 5, 9, 11, 12], "limit": [1, 3, 5, 12], "linear": 4, "linux": 10, "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "list_job": [2, 4], "live": [11, 12], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "local": [0, 13], "locat": [1, 6], "lock": 12, "log": [8, 10, 11, 12], "logic": [5, 12], "loop": 11, "ls_path": [2, 13], "maco": 11, "made": 3, "mai": [0, 9, 11], "main": 10, "make": [9, 11], "manag": [1, 5, 10], "mani": [8, 11], "manifest": [2, 8, 9, 10, 11, 12], "manifest_1": 10, "manifest_fil": [11, 12], "manifest_file_delimit": [11, 12], "manifest_row_pars": [11, 12], "map": [0, 11], "mark": 8, "master": 7, "match": [3, 5, 12], "max": 5, "max_concurrent_request": [2, 9, 11, 12], "max_presigned_url_ttl": 1, "max_tri": 8, "maximum": [11, 12], "md": [5, 7, 10, 12], "md5": [2, 3, 9, 11], "md5_hash": 11, "md5sum": [2, 9, 10], "mean": 8, "mechan": 3, "merg": 5, "metadata": [2, 3, 6, 9, 11], "metadata_list": 5, "metadata_sourc": 12, "metadata_typ": 12, "metdata": 12, "method": [1, 7, 10], "minimum": 10, "minut": 0, "mode": 7, "modul": [2, 10, 11], "more": [2, 5, 7, 9, 10], "most": 9, "mostli": 2, "multipl": [8, 11], "must": [1, 5], "my_common": 10, "my_credenti": 10, "my_field": 7, "my_index": 7, "my_program": 7, "my_project": 7, "name": [3, 4, 8, 10, 11, 12, 13], "namespac": [0, 12], "necessari": [3, 5], "need": [3, 7, 10, 11], "nest": 5, "net": 11, "never": 0, "new": [0, 3], "node": 8, "node_nam": 8, "node_typ": 8, "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "note": [0, 3, 11, 12], "noth": [3, 6], "now": [1, 8], "num": 5, "num_process": 11, "num_total_fil": 11, "number": [3, 7, 8, 11, 12], "o": 0, "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "object_id": [1, 2, 9, 10], "object_list": 10, "object_typ": [2, 9, 10], "objectid": 10, "obtain": [0, 10], "occur": 10, "off": 5, "offset": [5, 7], "oidc": 0, "old": 3, "one": [3, 5, 10, 11], "onli": [3, 5, 7, 8, 10, 11], "open": [8, 10, 11], "open_project": [2, 8], "openid": 0, "opt": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "order": [0, 8], "ordered_node_list": 8, "org": 10, "otherwis": 10, "output": [4, 5, 11, 12], "output_dir": 10, "output_filenam": [11, 12], "overrid": [0, 11, 12], "overwrit": 5, "own": 0, "packag": 10, "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "pagin": 3, "parallel": 11, "param": [3, 5, 8, 10], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "pars": [10, 11, 12, 13], "parser": [11, 12], "particular": 0, "pass": [0, 7, 8, 10], "password": [11, 12], "path": [0, 1, 5, 10, 11, 13], "path_to_manifest": 11, "pattern": [3, 12], "pdcdatastor": 11, "pend": 10, "per": [11, 12], "peregrin": 8, "permiss": 10, "persist": 9, "phs0001": 11, "phs0002": 11, "pick": 1, "pla": 11, "place": 11, "planx": 11, "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "popul": [10, 12], "posit": [1, 7], "possibl": 10, "post": [0, 11], "pprint": [2, 9, 10], "prefix": 3, "presign": 1, "pretti": 10, "prev_guid": [2, 9, 11], "previou": [3, 11], "previous": 4, "print": [8, 10], "process": 11, "processed_fil": 11, "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "program": [8, 11], "progress": [8, 10], "project": [8, 11], "project_id": [7, 8], "protocol": 1, "provid": [0, 1, 3, 5, 7, 8, 12], "public": [3, 5], "put": 0, "py": 11, "python": [2, 9, 11], "python3": 11, "python_subprocess_command": 11, "queri": [1, 2, 3, 5, 8, 11, 12], "query_str": 7, "query_txt": [7, 8], "query_url": [2, 3], "quickstart": 2, "rather": 0, "raw": [7, 11], "raw_data_download": [2, 7], "rbac": 3, "read": [3, 5, 11], "readm": 2, "reason": 10, "record": [1, 3, 5, 7, 8, 11, 12], "refresh": [0, 10], "refresh_access_token": [0, 2], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "regist": 8, "regular": 7, "relat": 9, "remov": [1, 6, 11, 13], "replac": 11, "replace_url": 11, "repo": 2, "repres": [3, 5, 10], "represent": [1, 3], "request": [0, 1, 3, 5, 8, 11, 12], "requir": 10, "resolv": 10, "resolve_object": [2, 9, 10], "respect": 7, "respons": [0, 1, 3, 4, 5], "result": [1, 8, 10, 11], "retri": 8, "retriev": [1, 8, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "return_full_metadata": 5, "rev": 3, "revers": 8, "revis": 3, "right": 1, "rm": [2, 13], "rm_path": [2, 13], "root": [11, 12], "row": [7, 8, 11, 12], "row_offset": 8, "rtype": 3, "run": [8, 11], "s3": [1, 10, 11], "safe": 11, "same": [5, 11, 13], "sampl": [8, 10], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "save": 10, "save_directori": 10, "schema": 8, "scope": [0, 1], "screen": 8, "script": 2, "search": [0, 2, 3], "second": [1, 4], "secret": 0, "see": [7, 10, 11], "self": 10, "semaphon": 12, "semaphor": 12, "separ": [0, 11], "server": 10, "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "service_loc": [3, 4, 5], "session": 11, "set": [0, 1, 5, 10], "setup": 2, "sheepdog": 8, "should": [0, 8, 11], "show": 10, "show_progress": 10, "shown": 11, "sign": 1, "signpost": 3, "similar": 10, "simpl": 3, "simpli": 11, "sinc": 3, "singl": [1, 5, 8], "size": [2, 3, 9, 10, 11], "skip": 8, "sleep": 4, "so": 11, "some": [0, 2], "someth": 11, "sort": 7, "sort_field": 7, "sort_object": 7, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "space": [0, 11], "specif": [5, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "spreadsheet": 8, "src_path": 13, "src_urlstr": 13, "src_w": 13, "src_wskei": 13, "ssl": [3, 4, 5], "start": [4, 7, 8, 10], "start_tim": [2, 9, 10], "static": 10, "statu": [2, 4, 9, 10], "storag": [1, 2, 6], "store": [1, 3, 10], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "string": [0, 3, 5, 11, 13], "strip": 11, "sub": 8, "subject": [7, 8], "submiss": 2, "submit": [8, 11], "submit_additional_metadata_column": 11, "submit_fil": [2, 8], "submit_record": [2, 8], "submitter_id": 7, "success": 10, "successfulli": 10, "suffici": 3, "suppli": 3, "support": [0, 1, 5, 8, 11], "sure": 1, "synchron": 11, "syntax": 7, "system": [6, 7, 8, 9], "t": [1, 5, 11], "tab": 11, "task": 9, "temporari": 11, "test": 11, "test1": 11, "test2": 11, "test3": 11, "test4": 11, "test5": 11, "text": [1, 7, 8], "than": [0, 5], "thei": [0, 10], "them": [10, 11], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "those": 11, "thread": 11, "thread_num": 11, "threadcontrol": [2, 9, 11], "through": [8, 11], "tier": 7, "time": [1, 3, 8, 10, 11], "timestamp": 10, "tmp_folder": [2, 9, 11], "token": [0, 10], "tool": 2, "total": 11, "treat": [1, 5], "tree": 10, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "try": 0, "tsv": [8, 11, 12], "tupl": [0, 3, 11, 12], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "typic": 10, "uc": 7, "unaccess": 7, "under": [0, 8, 13], "uniqu": [1, 5], "unknown": 10, "unpack": 10, "unpack_packag": 10, "until": [4, 10], "up": [5, 9], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "update_blank": [2, 3], "update_record": [2, 3], "updated_tim": [2, 9, 10], "upload": [1, 2, 3, 8, 13], "upload_fil": [1, 2], "upload_file_to_guid": [1, 2], "upload_url": [2, 13], "url": [1, 2, 3, 9, 10, 11, 12, 13], "urls_metadata": 3, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "usag": 11, "use_agg_md": 5, "user": [0, 10, 12], "user_access": [2, 9, 10], "usual": 12, "utcnow": 1, "util": 9, "uuid": [1, 3, 8], "uuid1": 8, "uuid2": 8, "valid": [3, 7], "valu": [0, 1, 3, 5, 7, 10, 11], "value_from_indexd": 11, "value_from_manifest": 11, "variabl": [0, 7, 8], "variou": 2, "verbos": [7, 8], "verif": 11, "verifi": [2, 9], "verify_manifest": 11, "verify_object_manifest": 11, "version": [3, 4, 5], "vital_statu": 7, "w": 13, "wa": 0, "wai": 10, "wait": 4, "want": [0, 3, 8], "warn": 11, "we": [3, 11], "web": 0, "what": 5, "when": [0, 3, 7, 10, 12], "where": [0, 3, 5, 11, 12], "whether": [3, 4, 5, 8, 11, 12], "which": [8, 10], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "whose": 5, "within": [0, 2, 9], "without": [3, 5], "won": 5, "work": [0, 10], "workaround": 11, "worksheet": 8, "workspac": [0, 2], "wrapper": 10, "write": 11, "ws_urlstr": 13, "wskei": 13, "wss": 13, "wt": [0, 10], "x": 11, "xlsx": 8, "you": [0, 3, 8, 11], "your": 0}, "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "titleterms": {"": 2, "auth": 0, "class": [1, 3, 4, 5, 6, 7, 8], "document": 2, "download": [10, 11], "dr": 10, "file": 1, "gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "helper": 0, "index": [3, 11], "indic": 2, "ingest": 12, "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "sdk": 2, "storag": 13, "submiss": 8, "tabl": 2, "tool": [9, 10, 11, 12], "verifi": 11, "welcom": 2, "workspac": 13}}) \ No newline at end of file +Search.setIndex({"alltitles": {"DRS Download Tools": [[10, null]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Gen3 Auth Helper": [[0, null]], "Gen3 File Class": [[1, null]], "Gen3 Index Class": [[3, null]], "Gen3 Jobs Class": [[4, null]], "Gen3 Metadata Class": [[5, null]], "Gen3 Object Class": [[6, null]], "Gen3 Query Class": [[7, null]], "Gen3 Submission Class": [[8, null]], "Gen3 Tools": [[9, null]], "Gen3 Workspace Storage": [[13, null]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Indexing Tools": [[11, null]], "Indices and tables": [[2, "indices-and-tables"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Metadata Tools": [[12, null]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, null]]}, "docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "indexentries": {"_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager", false]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods", false]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS", false]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create", false]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases", false]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record", false]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias", false]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases", false]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest", false]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get", false]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases", false]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record", false]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum", false]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page", false]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params", false]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest", false]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls", false]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd", false]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait", false]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update", false]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases", false]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record", false]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest", false]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider", false]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ", false]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create", false]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens", false]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children", false]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID", false]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url", false]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy", false]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create", false]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases", false]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank", false]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path", false]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job", false]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version", false]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list", false]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program", false]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project", false]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record", false]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time", false]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl", false]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR", false]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete", false]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias", false]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases", false]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids", false]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file", false]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations", false]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path", false]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node", false]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes", false]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object", false]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program", false]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project", false]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record", false]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record", false]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records", false]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download", false]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download", false]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download", false]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest", false]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single", false]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url", false]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable", false]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager", false]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus", false]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time", false]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint", false]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node", false]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record", false]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name", false]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name", false]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size", false]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size", false]], "gen3.tools": [[9, "module-gen3.tools", false]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download", false]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest", false]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest", false]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest", false]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest", false]], "gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth", false]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File", false]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index", false]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs", false]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata", false]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object", false]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query", false]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission", false]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage", false]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get", false]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get", false]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token", false]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts", false]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases", false]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records", false]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all", false]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node", false]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token", false]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema", false]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix", false]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths", false]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version", false]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output", false]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url", false]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs", false]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary", false]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest", false]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects", false]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record", false]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc", false]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records", false]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page", false]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats", false]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status", false]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls", false]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids", false]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version", false]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version", false]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version", false]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions", false]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params", false]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query", false]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID", false]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT", false]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT", false]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname", false]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest", false]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE", false]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy", false]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy", false]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy", false]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest", false]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object", false]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest", false]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs", false]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load", false]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest", false]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls", false]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path", false]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest", false]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS", false]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5", false]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum", false]], "module": [[9, "module-gen3.tools", false], [10, "module-gen3.tools.download.drs_download", false], [11, "module-gen3.tools.indexing.download_manifest", false], [11, "module-gen3.tools.indexing.index_manifest", false], [11, "module-gen3.tools.indexing.verify_manifest", false], [12, "module-gen3.tools.metadata.ingest_manifest", false]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id", false]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id", false]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type", false]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project", false]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint", false]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID", false]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query", false]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query", false]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query", false]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls", false]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download", false]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token", false]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects", false]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm", false]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path", false]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE", false]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time", false]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status", false]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file", false]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record", false]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl", false]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER", false]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update", false]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases", false]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank", false]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record", false]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time", false]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload", false]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file", false]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid", false]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url", false]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS", false]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access", false]]}, "objects": {"gen3": [[9, 3, 0, "-", "tools"]], "gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "terms": {"": [1, 4, 8, 10, 11], "0": [5, 7, 8, 11], "0084667": [], "013444": [], "0248635": [], "0791543": [], "0a80fada010c": 11, "0a80fada096c": 11, "0a80fada097c": 11, "0a80fada098c": 11, "0a80fada099c": 11, "1": [4, 5, 8, 10, 11], "10": [5, 7, 11], "100": [8, 11], "1125998": 12, "11e9": 11, "120715": [], "1329331": [], "1468077": [], "165499": [], "1724170637": [], "1726237606": [], "1726237962": [], "1728059996": [], "1728059997": [], "1728061593": [], "1728061594": [], "1729113765": [], "1729629354": [], "1729629627": [], "1732401632": [], "1732401820": [], "1732401821": [], "1733254905": [], "1733263055": [], "1733431971": [], "1733932163": [], "1734031012": [], "1734031066": [], "1734031067": [], "1734474422": [], "1734474533": 11, "1734474534": 12, "2": [5, 11], "2023835": [], "2310405": [], "24": [11, 12], "244287": [], "2511687": [], "255e396f": 11, "256328": [], "284675": [], "3": [5, 11], "30": 8, "313695": [], "33": 5, "333": 5, "3410327": [], "343434344": 11, "363455714": 11, "3686683": [], "3697538": [], "3759608": [], "4": [5, 11], "4401503": [], "450c": 11, "4714": 8, "473d83400bc1bc9dc635e334fadd433c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "5": [0, 5], "50": 7, "5117643": [], "518752": [], "543434443": 11, "5833347": [], "6129959": [], "6138082": [], "6146133": [], "653854": [], "6f90": 8, "7d3d8d2083b4": 11, "8366766": [], "8420": 8, "848236": [], "8493605": [], "8535511": [], "863009": 11, "8712757": [], "9082778": [], "93d9af72": 11, "998105": [], "9a07": 11, "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "AND": 5, "Be": 1, "But": 5, "By": 11, "For": [1, 5, 6, 7, 8, 9, 11], "IF": 11, "If": [0, 1, 7, 11, 12], "In": 10, "It": 10, "NOT": 12, "OR": 5, "One": 7, "Such": 9, "THE": 11, "That": 3, "The": [0, 1, 2, 3, 5, 8, 10, 11], "There": 11, "These": 9, "To": 11, "Will": [4, 7, 10], "_get_acl_from_row": 11, "_get_authz_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_for_row": 12, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "_guid_typ": 12, "_manag": [2, 9, 10], "_query_for_associated_indexd_record_guid": 12, "_ssl": [3, 4, 5], "a5c6": 11, "ab167e49d25b488939b1ede42752458b": 3, "about": [2, 3], "abov": 11, "access": [0, 1, 3, 7, 10], "access_method": [2, 9, 10], "access_token": 0, "accesstoken": 0, "acl": [2, 3, 9, 11], "across": 11, "act": 0, "action": [9, 11], "actual": 11, "ad": 3, "add": [3, 5], "addit": [3, 5, 10, 11], "admin": [5, 11], "admin_endpoint_suffix": 5, "after": 10, "against": [3, 7, 8, 11, 12], "algorithm": 3, "alia": [3, 5], "alias": 5, "aliv": 7, "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "allow": [0, 6, 8, 10, 11, 12], "allowed_data_upload_bucket": 1, "along": 2, "alreadi": 9, "also": 1, "altern": [5, 11], "alwai": 5, "ammount": 12, "amount": [1, 9], "an": [0, 3, 6, 8, 10, 11], "ani": [0, 5, 10, 11], "anoth": 5, "api": [0, 5, 8, 11], "api_kei": 11, "appli": 7, "appropri": 13, "ar": [5, 7, 8, 9, 10, 11], "arbitrari": 0, "argument": [0, 13], "arrai": 8, "asc": 7, "assign": 9, "assist": 10, "associ": [3, 5], "assum": 11, "async": [3, 4, 5, 9, 11, 12], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_create_record": [2, 3], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_download_object_manifest": [2, 9, 11], "async_get": [2, 5], "async_get_alias": [2, 5], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_url": [2, 3], "async_query_urls_from_indexd": [2, 9, 12], "async_run_job_and_wait": [2, 4], "async_upd": [2, 5], "async_update_alias": [2, 5], "async_update_record": [2, 3], "async_verify_object_manifest": [2, 9, 11], "asynchron": [3, 4, 5], "asyncio": [11, 12], "asyncron": 5, "attach": [3, 5], "attempt": 11, "attribut": [10, 11], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "authbas": 0, "authent": 0, "author": 1, "authz": [0, 1, 2, 3, 9, 10, 11], "auto": [0, 2], "automat": 0, "avail": [1, 2, 10, 11], "az": 1, "b": [5, 11], "b0f1": 11, "bar": 10, "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "baseid": 3, "basic": [3, 11, 12], "batch_creat": [2, 5], "batch_siz": 8, "behalf": 0, "behavior": 11, "being": 3, "belong": 8, "below": 11, "blank": 3, "blob": [5, 7], "bodi": 3, "bool": [4, 5, 8, 10, 11, 12], "boolean": 3, "bownload": 10, "broad": 9, "broken": 9, "bucket": [1, 6], "bundl": 10, "byte": 10, "c": [5, 11], "cach": 10, "cache_hosts_wts_token": [2, 9, 10], "call": [10, 13], "can": [0, 3, 4, 8, 11, 12], "capabl": 9, "case": [0, 10], "categori": 9, "ccle": 8, "ccle_one_record": 8, "ccle_sample_nod": 8, "cdi": 7, "chang": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "child": 10, "children": [2, 9, 10], "chunk_siz": 8, "class": [0, 2, 10, 11, 13], "cli": 10, "client": [0, 3], "client_credenti": 0, "client_id": 0, "client_scop": 0, "client_secret": 0, "code": [2, 8], "column": [11, 12], "column_to_use_as_guid": [2, 9, 12], "columna": 11, "columnb": 11, "columnc": 11, "com": 7, "comma": 11, "command": [10, 11], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "commons_url": [2, 9, 10, 11, 12], "complet": [4, 11], "complex": 7, "concat": 11, "concurr": [11, 12], "configur": 1, "connect": 12, "consist": 3, "constructor": 0, "contain": [0, 2, 5, 8, 9, 10, 11, 12], "content": [3, 13], "content_created_d": 3, "content_updated_d": 3, "continu": 10, "control": 3, "conveni": 10, "copi": [2, 13], "coroutin": 11, "correspond": 3, "count": 3, "crdc": 0, "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_blank": [2, 3], "create_index_key_path": [2, 5], "create_job": [2, 4], "create_new_vers": [2, 3], "create_object_list": [2, 9, 10], "create_program": [2, 8], "create_project": [2, 8], "create_record": [2, 3], "created_tim": [2, 9, 10], "creation": [3, 11], "cred": 3, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "csv": [8, 11, 12], "curl": [0, 2], "current": [6, 8, 10], "current_dir": [2, 9, 11], "custom": 11, "d": 5, "d70b41b9": 8, "data": [0, 1, 3, 5, 7, 8, 10, 11], "data_spreadsheet": 8, "data_typ": 7, "data_upload_bucket": 1, "dataa": 11, "datab": 11, "databas": 5, "datacommon": 0, "datafil": 10, "datamanag": 10, "date": 3, "datetim": [1, 3, 10], "dbgap": 12, "dcf": 8, "def": 11, "default": [0, 1, 3, 7, 8, 11, 12], "defin": [5, 8, 10], "delai": 4, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_all_guid": [2, 9, 11], "delete_fil": [1, 2], "delete_file_loc": [1, 2, 6], "delete_index_key_path": [2, 5], "delete_nod": [2, 8], "delete_object": [2, 6], "delete_program": [2, 8], "delete_project": [2, 8], "delete_record": [2, 3, 8], "delete_unpacked_packag": 10, "delimet": [11, 12], "delimit": 11, "demograph": 8, "deprec": 1, "descript": [3, 5], "desir": 11, "dest_path": 13, "dest_urlstr": 13, "dest_w": 13, "dest_wskei": 13, "detail": [2, 7, 10], "determin": [10, 11, 12], "dev": 11, "dict": [3, 4, 5, 10, 11, 12], "dictionari": [3, 4, 5, 7, 8], "did": 3, "differ": 5, "direct": 0, "directori": [10, 11], "disabl": 10, "discoveri": 10, "disk": 13, "dispatch": 4, "dist_resolut": 3, "distribut": 3, "doc": [7, 10], "docstr": 2, "document": [1, 3], "doe": [0, 12], "domain": [11, 12], "done": 4, "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "download_files_in_drs_manifest": [2, 9, 10], "download_list": 10, "download_manifest": 11, "download_singl": [1, 2], "download_url": [2, 13], "downloadmanag": [2, 9, 10], "downloadstatu": [2, 9, 10], "dr": [2, 9], "drs_download": 10, "drs_hostnam": 10, "drsdownload": 10, "drsobjecttyp": 10, "e": [5, 10], "e043ab8b77b9": 8, "each": [3, 8, 10, 11], "effici": 9, "eg": 3, "either": 8, "elasticsearch": 7, "els": [0, 12], "elsewher": 12, "empti": [8, 11], "enabl": 11, "end": [5, 10], "end_tim": [2, 9, 10], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 10, 13], "entir": 8, "entri": [3, 11], "env": 0, "environ": 0, "equal": 7, "error": [10, 11, 12], "error_nam": 11, "etc": 8, "even": 11, "everi": [9, 11], "everyth": 11, "ex": [0, 11, 12], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "exclud": 3, "execut": [7, 8, 11], "exist": [1, 3, 5, 6, 9, 12], "expect": [5, 9, 11], "experi": 8, "expir": [0, 1], "expires_in": 1, "export": [8, 10], "export_nod": [2, 8], "export_record": [2, 8], "f1f8": 11, "factori": 10, "fail": [8, 10], "fals": [3, 5, 6, 10, 11], "featur": [1, 6], "fenc": [0, 1], "fetch": 0, "field": [3, 5, 7, 11, 12], "fieldnam": 11, "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "file_nam": [1, 2, 3, 9, 10, 11], "file_s": [2, 9, 10, 11], "file_st": 3, "fileformat": 8, "filenam": [0, 8, 10, 11, 12], "files": 10, "fill": 12, "filter": [5, 7], "filter_object": 7, "first": [7, 8], "flag": 11, "folder": 11, "follow": [0, 11], "forc": 11, "force_metadata_columns_even_if_empti": 11, "form": 13, "format": [3, 5, 8, 11], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "func_to_parse_row": [11, 12], "function": [2, 3, 4, 5, 9, 10, 11, 12], "g": 10, "gen3": [10, 11, 12], "gen3_api_kei": 0, "gen3_oidc_client_creds_secret": 0, "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "gen3fil": [1, 2], "gen3index": [2, 3], "gen3job": [2, 4, 10], "gen3metadata": [2, 5], "gen3object": [2, 6], "gen3queri": [2, 7], "gen3submiss": [2, 8], "gen3wsstorag": [2, 13], "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "get_access_token": [0, 2], "get_access_token_from_wt": [0, 2], "get_alias": [2, 5], "get_all_record": [2, 3], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_fresh_token": [2, 9, 10], "get_graphql_schema": [2, 8], "get_guid_from_fil": 12, "get_guids_prefix": [2, 3], "get_index_key_path": [2, 5], "get_latest_vers": [2, 3], "get_output": [2, 4], "get_presigned_url": [1, 2], "get_program": [2, 8], "get_project": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_statu": [2, 4], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "giangb": 11, "github": [2, 7], "give": 1, "given": [0, 3, 4, 5, 8, 10, 12, 13], "global": [4, 5], "good": 3, "grant": 0, "graph": 8, "graphql": [7, 8], "graphql_queri": [2, 7], "group": 3, "guid": [1, 2, 3, 5, 6, 9, 11, 12], "guid_exampl": 11, "guid_for_row": 12, "guid_from_fil": 12, "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "guppi": 7, "ha": [0, 11], "handl": [3, 10], "hardcod": 0, "has_vers": 3, "hash": [3, 11], "hash_typ": 3, "have": [5, 11], "header": 11, "healthi": [3, 4, 5], "help": 11, "helper": 2, "hit": 11, "host": 10, "hostnam": [2, 9, 10], "how": [8, 11], "howto": 10, "http": [0, 7, 11, 12], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "id": [0, 1, 3, 5, 10, 11], "idea": 3, "identifi": [3, 5, 9, 11], "idp": 0, "illustr": 11, "immut": 3, "implement": 0, "implic": 11, "import": 11, "includ": [0, 3], "indent": 10, "index": [0, 2, 5, 9], "index_manifest": 11, "index_object_manifest": [2, 9, 11], "indexd": [1, 3, 6, 10, 11, 12], "indexd_field": [11, 12], "indexd_record_page_s": [2, 9, 11], "indexed_file_object_guid": 12, "indic": [0, 11], "infil": 10, "info": [3, 11], "inform": [2, 3, 10], "ingest": [2, 9], "ingest_manifest": 12, "initi": [0, 10], "input": [4, 10, 11], "input_manifest": 11, "instal": [0, 2, 11], "instanc": [1, 3, 6, 7, 8, 9, 10], "instead": [1, 7, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "integ": [1, 3, 8], "intend": 0, "interact": [1, 3, 4, 5, 6, 8, 13], "interest": 10, "interpret": 0, "introspect": 8, "involv": 9, "is_healthi": [2, 3, 4, 5], "is_indexed_file_object": 12, "isn": 1, "issu": 0, "its": [1, 3], "job": 2, "job_id": 4, "job_input": 4, "job_nam": 4, "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "just": [5, 11, 12], "jwt": 0, "kei": [0, 3, 5, 13], "know": 11, "known": 10, "kwarg": [3, 4, 5], "l": [2, 13], "larg": 9, "last": 10, "latest": 3, "least": 3, "level": 6, "librari": 11, "like": [3, 5, 9, 11, 12], "limit": [1, 3, 5, 12], "linear": 4, "linux": 10, "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "list_job": [2, 4], "live": [11, 12], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "local": [0, 13], "locat": [1, 6], "lock": 12, "log": [8, 10, 11, 12], "logic": [5, 12], "loop": 11, "ls_path": [2, 13], "maco": 11, "made": 3, "mai": [0, 9, 11], "main": 10, "make": [9, 11], "manag": [1, 5, 10], "mani": [8, 11], "manifest": [2, 8, 9, 10, 11, 12], "manifest_1": 10, "manifest_fil": [11, 12], "manifest_file_delimit": [11, 12], "manifest_row_pars": [11, 12], "map": [0, 11], "mark": 8, "master": 7, "match": [3, 5, 12], "max": 5, "max_concurrent_request": [2, 9, 11, 12], "max_presigned_url_ttl": 1, "max_tri": 8, "maximum": [11, 12], "md": [5, 7, 10, 12], "md5": [2, 3, 9, 11], "md5_hash": 11, "md5sum": [2, 9, 10], "mean": 8, "mechan": 3, "merg": 5, "metadata": [2, 3, 6, 9, 11], "metadata_list": 5, "metadata_sourc": 12, "metadata_typ": 12, "metdata": 12, "method": [1, 7, 10], "minimum": 10, "minut": 0, "mode": 7, "modul": [2, 10, 11], "more": [2, 5, 7, 9, 10], "most": 9, "mostli": 2, "multipl": [8, 11], "must": [1, 5], "my_common": 10, "my_credenti": 10, "my_field": 7, "my_index": 7, "my_program": 7, "my_project": 7, "name": [3, 4, 8, 10, 11, 12, 13], "namespac": [0, 12], "necessari": [3, 5], "need": [3, 7, 10, 11], "nest": 5, "net": 11, "never": 0, "new": [0, 3], "node": 8, "node_nam": 8, "node_typ": 8, "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "note": [0, 3, 11, 12], "noth": [3, 6], "now": [1, 8], "num": 5, "num_process": 11, "num_total_fil": 11, "number": [3, 7, 8, 11, 12], "o": 0, "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "object_id": [1, 2, 9, 10], "object_list": 10, "object_typ": [2, 9, 10], "objectid": 10, "obtain": [0, 10], "occur": 10, "off": 5, "offset": [5, 7], "oidc": 0, "old": 3, "one": [3, 5, 10, 11], "onli": [3, 5, 7, 8, 10, 11], "open": [8, 10, 11], "open_project": [2, 8], "openid": 0, "opt": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "order": [0, 8], "ordered_node_list": 8, "org": 10, "otherwis": 10, "output": [4, 5, 11, 12], "output_dir": 10, "output_filenam": [11, 12], "overrid": [0, 11, 12], "overwrit": 5, "own": 0, "packag": 10, "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "pagin": 3, "parallel": 11, "param": [3, 5, 8, 10], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "pars": [10, 11, 12, 13], "parser": [11, 12], "particular": 0, "pass": [0, 7, 8, 10], "password": [11, 12], "path": [0, 1, 5, 10, 11, 13], "path_to_manifest": 11, "pattern": [3, 12], "pdcdatastor": 11, "pend": 10, "per": [11, 12], "peregrin": 8, "permiss": 10, "persist": 9, "phs0001": 11, "phs0002": 11, "pick": 1, "pla": 11, "place": 11, "planx": 11, "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "popul": [10, 12], "posit": [1, 7], "possibl": 10, "post": [0, 11], "pprint": [2, 9, 10], "prefix": 3, "presign": 1, "pretti": 10, "prev_guid": [2, 9, 11], "previou": [3, 11], "previous": 4, "print": [8, 10], "process": 11, "processed_fil": 11, "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "program": [8, 11], "progress": [8, 10], "project": [8, 11], "project_id": [7, 8], "protocol": 1, "provid": [0, 1, 3, 5, 7, 8, 12], "public": [3, 5], "put": 0, "py": 11, "python": [2, 9, 11], "python3": 11, "python_subprocess_command": 11, "queri": [1, 2, 3, 5, 8, 11, 12], "query_str": 7, "query_txt": [7, 8], "query_url": [2, 3], "quickstart": 2, "rather": 0, "raw": [7, 11], "raw_data_download": [2, 7], "rbac": 3, "read": [3, 5, 11], "readm": 2, "reason": 10, "record": [1, 3, 5, 7, 8, 11, 12], "refresh": [0, 10], "refresh_access_token": [0, 2], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "regist": 8, "regular": 7, "relat": 9, "remov": [1, 6, 11, 13], "replac": 11, "replace_url": 11, "repo": 2, "repres": [3, 5, 10], "represent": [1, 3], "request": [0, 1, 3, 5, 8, 11, 12], "requir": 10, "resolv": 10, "resolve_object": [2, 9, 10], "respect": 7, "respons": [0, 1, 3, 4, 5], "result": [1, 8, 10, 11], "retri": 8, "retriev": [1, 8, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "return_full_metadata": 5, "rev": 3, "revers": 8, "revis": 3, "right": 1, "rm": [2, 13], "rm_path": [2, 13], "root": [11, 12], "row": [7, 8, 11, 12], "row_offset": 8, "rtype": 3, "run": [8, 11], "s3": [1, 10, 11], "safe": 11, "same": [5, 11, 13], "sampl": [8, 10], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "save": 10, "save_directori": 10, "schema": 8, "scope": [0, 1], "screen": 8, "script": 2, "search": [0, 2, 3], "second": [1, 4], "secret": 0, "see": [7, 10, 11], "self": 10, "semaphon": 12, "semaphor": 12, "separ": [0, 11], "server": 10, "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "service_loc": [3, 4, 5], "session": 11, "set": [0, 1, 5, 10], "setup": 2, "sheepdog": 8, "should": [0, 8, 11], "show": 10, "show_progress": 10, "shown": 11, "sign": 1, "signpost": 3, "similar": 10, "simpl": 3, "simpli": 11, "sinc": 3, "singl": [1, 5, 8], "size": [2, 3, 9, 10, 11], "skip": 8, "sleep": 4, "so": 11, "some": [0, 2], "someth": 11, "sort": 7, "sort_field": 7, "sort_object": 7, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "space": [0, 11], "specif": [5, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "spreadsheet": 8, "src_path": 13, "src_urlstr": 13, "src_w": 13, "src_wskei": 13, "ssl": [3, 4, 5], "start": [4, 7, 8, 10], "start_tim": [2, 9, 10], "static": 10, "statu": [2, 4, 9, 10], "storag": [1, 2, 6], "store": [1, 3, 10], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "string": [0, 3, 5, 11, 13], "strip": 11, "sub": 8, "subject": [7, 8], "submiss": 2, "submit": [8, 11], "submit_additional_metadata_column": 11, "submit_fil": [2, 8], "submit_record": [2, 8], "submitter_id": 7, "success": 10, "successfulli": 10, "suffici": 3, "suppli": 3, "support": [0, 1, 5, 8, 11], "sure": 1, "synchron": 11, "syntax": 7, "system": [6, 7, 8, 9], "t": [1, 5, 11], "tab": 11, "task": 9, "temporari": 11, "test": 11, "test1": 11, "test2": 11, "test3": 11, "test4": 11, "test5": 11, "text": [1, 7, 8], "than": [0, 5], "thei": [0, 10], "them": [10, 11], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "those": 11, "thread": 11, "thread_num": 11, "threadcontrol": [2, 9, 11], "through": [8, 11], "tier": 7, "time": [1, 3, 8, 10, 11], "timestamp": 10, "tmp_folder": [2, 9, 11], "token": [0, 10], "tool": 2, "total": 11, "treat": [1, 5], "tree": 10, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "try": 0, "tsv": [8, 11, 12], "tupl": [0, 3, 11, 12], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "typic": 10, "uc": 7, "unaccess": 7, "under": [0, 8, 13], "uniqu": [1, 5], "unknown": 10, "unpack": 10, "unpack_packag": 10, "until": [4, 10], "up": [5, 9], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "update_blank": [2, 3], "update_record": [2, 3], "updated_tim": [2, 9, 10], "upload": [1, 2, 3, 8, 13], "upload_fil": [1, 2], "upload_file_to_guid": [1, 2], "upload_url": [2, 13], "url": [1, 2, 3, 9, 10, 11, 12, 13], "urls_metadata": 3, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "usag": 11, "use_agg_md": 5, "user": [0, 10, 12], "user_access": [2, 9, 10], "usual": 12, "utcnow": 1, "util": 9, "uuid": [1, 3, 8], "uuid1": 8, "uuid2": 8, "valid": [3, 7], "valu": [0, 1, 3, 5, 7, 10, 11], "value_from_indexd": 11, "value_from_manifest": 11, "variabl": [0, 7, 8], "variou": 2, "verbos": [7, 8], "verif": 11, "verifi": [2, 9], "verify_manifest": 11, "verify_object_manifest": 11, "version": [3, 4, 5], "vital_statu": 7, "w": 13, "wa": 0, "wai": 10, "wait": 4, "want": [0, 3, 8], "warn": 11, "we": [3, 11], "web": 0, "what": 5, "when": [0, 3, 7, 10, 12], "where": [0, 3, 5, 11, 12], "whether": [3, 4, 5, 8, 11, 12], "which": [8, 10], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "whose": 5, "within": [0, 2, 9], "without": [3, 5], "won": 5, "work": [0, 10], "workaround": 11, "worksheet": 8, "workspac": [0, 2], "wrapper": 10, "write": 11, "ws_urlstr": 13, "wskei": 13, "wss": 13, "wt": [0, 10], "x": 11, "xlsx": 8, "you": [0, 3, 8, 11], "your": 0}, "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "titleterms": {"": 2, "auth": 0, "class": [1, 3, 4, 5, 6, 7, 8], "document": 2, "download": [10, 11], "dr": 10, "file": 1, "gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "helper": 0, "index": [3, 11], "indic": 2, "ingest": 12, "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "sdk": 2, "storag": 13, "submiss": 8, "tabl": 2, "tool": [9, 10, 11, 12], "verifi": 11, "welcom": 2, "workspac": 13}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index a3b8ff357..b5d60795f 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -381,7 +381,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1734474422.6146133.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1734474533.863009.log')[source]

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index 3f9430065..3f47dd6f8 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1734474422.8366766.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1734474534.1125998.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters:
From 2f2b21370568f9a7c95c227ad53e4d6ef5c1d0e8 Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Thu, 19 Dec 2024 15:43:03 -0600 Subject: [PATCH 5/8] make requested changes --- .../indexing/post_indexing_validation.py | 99 +++++++++++-------- tests/test_post_indexing_validation.py | 79 +++++++++++---- 2 files changed, 117 insertions(+), 61 deletions(-) diff --git a/gen3/tools/indexing/post_indexing_validation.py b/gen3/tools/indexing/post_indexing_validation.py index 3b3d1d32a..5d3c50ad4 100644 --- a/gen3/tools/indexing/post_indexing_validation.py +++ b/gen3/tools/indexing/post_indexing_validation.py @@ -13,6 +13,7 @@ from cdislogging import get_logger, get_stream_handler from gen3.file import Gen3File import requests +import re logger = get_logger(__name__) @@ -24,20 +25,19 @@ class GuidError(Exception): pass +class ManifestError(Exception): + pass + + class Record: - def __init__(self, guid, bucket, protocol, acl, access_token, commons, size): + def __init__(self, guid, bucket, protocol, acl, size): self.guid = guid self.bucket = bucket self.protocol = protocol self.acl = acl - self.commons = commons self.size = size self.response_status = -1 self.download_status = -1 - self.headers = { - "accept": "application/json", - "authorization": f"bearer {access_token}", - } def check_record(self, gen3file): """ @@ -70,30 +70,25 @@ def check_record(self, gen3file): logger.info(f"Pre-signed url generation failed for record {self.guid}") self.response_status = response_status - download_success = -1 if response_status == 200: try: - download_success = requests.get(url).status_code + self.download_status = requests.get(url).status_code logger.info( - f"Download process complete with status code {download_success}" + f"Download process complete with status code {self.download_status}" ) - except: - download_success = -1 - self.download_status = download_success + except Exception as err: + self.download_status = -1 + logger.info( + f"Download process failed due to {err}" + ) # maybe should be logger.error return -class Records: +class RecordParser: def __init__(self, auth): self.auth = auth - self.access_token = auth.get_access_token() - self.commons = auth.endpoint self.record_dict = {} self.record_sizes = {} - self.headers = { - "accept": "application/json", - "authorization": f"bearer {self.access_token}", - } def read_records_from_manifest(self, manifest): """ @@ -102,10 +97,17 @@ def read_records_from_manifest(self, manifest): Args: manifest (str): the location of a manifest file """ - if manifest[-3:] == "tsv": + tsv_pattern = "^.*\.tsv$" + csv_pattern = "^.*\.csv$" + if re.match(tsv_pattern, manifest): sep = "\t" - else: + elif re.match(csv_pattern, manifest): sep = "," + else: + raise ManifestError( + "Please enter the path to a valid manifest in .csv or .tsv format" + ) + with open(manifest, mode="r") as f: csv_reader = csv.DictReader(f, delimiter=sep) rows = [row for row in csv_reader] @@ -118,6 +120,7 @@ def read_records_from_manifest(self, manifest): "Manifest file has no column named 'GUID', 'guid', or 'id'" ) + url_pattern = r"^[a-zA-Z][a-zA-Z0-9+.-]*://[^\s/]+(?:/[^\s]*)*$" for row in rows: url_parsed = False size = row["size"] @@ -125,25 +128,37 @@ def read_records_from_manifest(self, manifest): for acl in row["acl"].split(" "): if acl != "admin": for url in row["url"].split(" "): - if "://" not in url: - continue + url = url.replace("[", "").replace("]", "") + if re.match(url_pattern, url) == None: + raise ManifestError( + f"Manifest contains guid {guid} with invalid url format: {url}" + ) else: protocol, bucket = ( url.split("://")[0].replace("[", ""), url.split("/")[2], ) - key = (bucket, protocol, acl) - if key not in self.record_dict or ( - int(self.record_dict[key].size) >= int(size) - and int(size) != 0 - ): + key = ( + bucket, + protocol, + acl, + ) # Check a record for each unique (bucket, protocol, acl) combination + if ( + key not in self.record_dict + or ( # Add record to the list of records if no matching (bucket,protocol,acl) found + int(self.record_dict[key].size) + >= int( + size + ) # Update to download smallest non-zero sized record + and int(size) + != 0 # Make sure record has non-zero size + ) + ): # If it passes these we temporarily choose this record to check for the bucket, protocol, and acl record = Record( guid, bucket, protocol, acl, - self.access_token, - self.commons, size, ) self.record_dict[key] = record @@ -179,9 +194,9 @@ def save_download_check_results_to_csv(self, csv_filename): Args: csv_filename (str): the relative file path of the output csv """ - download_results = [] + self.download_results = [] for record in self.record_dict.values(): - download_results.append( + self.download_results.append( { "acl": record.acl, "bucket": record.bucket, @@ -192,10 +207,10 @@ def save_download_check_results_to_csv(self, csv_filename): } ) - self.download_results = download_results + self.download_results = self.download_results # Check if the results list is empty - if not download_results: + if not self.download_results: logger.warning("No results to save.") return @@ -216,15 +231,15 @@ def save_download_check_results_to_csv(self, csv_filename): writer.writeheader() # Iterate through the DownloadCheckResult instances and write each row - for result in download_results: + for result in self.download_results: writer.writerow( { - "ACL": result["acl"], - "Bucket": result["bucket"], - "Protocol": result["protocol"], - "Presigned URL Status": result["presigned_url_status"], - "Download Status": result["download_status"], - "GUID": result["guid"], + "acl": result["acl"], + "bucket": result["bucket"], + "protocol": result["protocol"], + "presigned_url_status": result["presigned_url_status"], + "download_status": result["download_status"], + "guid": result["guid"], } ) @@ -243,7 +258,7 @@ def validate_manifest(MANIFEST, auth, output_file="results.csv"): auth (str): a Gen3Auth instance """ logger.info("Starting...") - records = Records(auth) + records = RecordParser(auth) records.read_records_from_manifest(MANIFEST) records.check_records() records.save_download_check_results_to_csv(output_file) diff --git a/tests/test_post_indexing_validation.py b/tests/test_post_indexing_validation.py index dbea6c91a..e3de4d540 100644 --- a/tests/test_post_indexing_validation.py +++ b/tests/test_post_indexing_validation.py @@ -1,13 +1,11 @@ import pytest import os from gen3.tools.indexing.post_indexing_validation import ( - Record, - Records, validate_manifest, + ManifestError, ) from unittest import mock -from unittest.mock import MagicMock, AsyncMock, mock_open -from aiohttp import ClientResponse +from unittest.mock import MagicMock, mock_open import gen3 import time import base64 @@ -16,7 +14,8 @@ cwd = os.path.dirname(os.path.realpath(__file__)) -read_data = "guid,md5,size,acl,url\n255e396f-f1f8-11e9-9a07-0a80fada099c,473d83400bc1bc9dc635e334faddf33d,363455714,['Open'],['s3://pdcdatastore/test1.raw']\n255e396f-f1f8-11e9-9a07-0a80fada098d,473d83400bc1bc9dc635e334faddd33c,343434344,['Open'],['s3://pdcdatastore/test2.raw']\n255e396f-f1f8-11e9-9a07-0a80fada097c,473d83400bc1bc9dc635e334fadd433c,543434443,['phs0001'],['s3://pdcdatastore/test3.raw']" +read_data = "guid,md5,size,acl,url\n255e396f-f1f8-11e9-9a07-0a80fada099c,473d83400bc1bc9dc635e334faddf33d,363455714,[Open],[s3://pdcdatastore/test1.raw]\n255e396f-f1f8-11e9-9a07-0a80fada098d,473d83400bc1bc9dc635e334faddd33c,343434344,[Open],[s3://pdcdatastore/test2.raw]\n255e396f-f1f8-11e9-9a07-0a80fada097c,473d83400bc1bc9dc635e334fadd433c,543434443,[phs0001],[s3://pdcdatastore/test3.raw]" +read_data2 = "guid,md5,size,acl,url\n255e396f-f1f8-11e9-9a07-0a80fada099c,473d83400bc1bc9dc635e334faddf33d,363455714,[Open],[s3://pdcdatastore/test1.raw]\n255e396f-f1f8-11e9-9a07-0a80fada098d,473d83400bc1bc9dc635e334faddd33c,343434344,[Open],[s3://pdcdatastore/test2.raw]\n255e396f-f1f8-11e9-9a07-0a80fada097c,473d83400bc1bc9dc635e334fadd433c,543434443,[phs0001],[invalid_format]" class MockResponse: @@ -76,7 +75,6 @@ def test_validate_manifest_coro_with_200(): output = f"{cwd}/test_data/post_indexing_output.csv" auth = gen3.auth.Gen3Auth(refresh_token=test_key) - auth.get_access_token() records = validate_manifest(input, auth, output) mock_file_open.assert_called_with( @@ -85,12 +83,12 @@ def test_validate_manifest_coro_with_200(): mock_writeheader.assert_called_once() mock_writerow.assert_called_with( { - "ACL": "['phs0001']", - "Bucket": "pdcdatastore", - "Protocol": "'s3", - "Presigned URL Status": 200, - "Download Status": 200, - "GUID": "255e396f-f1f8-11e9-9a07-0a80fada097c", + "acl": "[phs0001]", + "bucket": "pdcdatastore", + "protocol": "s3", + "presigned_url_status": 200, + "download_status": 200, + "guid": "255e396f-f1f8-11e9-9a07-0a80fada097c", } ) @@ -148,7 +146,6 @@ def test_validate_manifest_coro_with_401(): output = f"{cwd}/test_data/post_indexing_output.csv" auth = gen3.auth.Gen3Auth(refresh_token=test_key) - auth.get_access_token() records = validate_manifest(input, auth, output) mock_file_open.assert_called_with( @@ -157,12 +154,12 @@ def test_validate_manifest_coro_with_401(): mock_writeheader.assert_called_once() mock_writerow.assert_called_with( { - "ACL": "['phs0001']", - "Bucket": "pdcdatastore", - "Protocol": "'s3", - "Presigned URL Status": 401, - "Download Status": -1, - "GUID": "255e396f-f1f8-11e9-9a07-0a80fada097c", + "acl": "[phs0001]", + "bucket": "pdcdatastore", + "protocol": "s3", + "presigned_url_status": 401, + "download_status": -1, + "guid": "255e396f-f1f8-11e9-9a07-0a80fada097c", } ) @@ -172,3 +169,47 @@ def test_validate_manifest_coro_with_401(): presigned_statuses.append(record["presigned_url_status"]) download_statuses.append(record["download_status"]) assert set(presigned_statuses) == {401} and set(download_statuses) == {-1} + + +def test_validate_manifest_coro_with_invalid_url(): + hostname = "test.datacommons.io" + exp = time.time() + 300 + decoded_info = {"aud": "123", "exp": exp, "iss": f"http://{hostname}"} + test_key = { + "api_key": "whatever." # pragma: allowlist secret + + base64.urlsafe_b64encode( + ('{"iss": "http://%s", "exp": %d }' % (hostname, exp)).encode("utf-8") + ).decode("utf-8") + + ".whatever" + } + get_mock = MockResponse(200) + with mock.patch( + "gen3.auth.get_access_token_with_key" + ) as mock_access_token, mock.patch( + "gen3.auth.Gen3Auth._write_to_file" + ) as mock_write_to_file, mock.patch( + "gen3.auth.decode_token" + ) as mock_decode_token, mock.patch( + "requests.get" + ) as mock_request, mock.patch( + "csv.DictWriter.writerow" + ) as mock_writerow, mock.patch( + "csv.DictWriter.writeheader" + ) as mock_writeheader, mock.patch( + "builtins.open", mock_open(read_data=read_data2) + ) as mock_file_open: + mock_access_token.return_value = "new_access_token" + mock_write_to_file().return_value = True + mock_decode_token.return_value = decoded_info + mock_request.return_value = get_mock + mock_writerow.return_value = None + mock_writeheader.return_value = None + + input = f"{cwd}/test_data/manifest3.csv" + output = f"{cwd}/test_data/post_indexing_output.csv" + + auth = gen3.auth.Gen3Auth(refresh_token=test_key) + try: + records = validate_manifest(input, auth, output) + except ManifestError: + assert True From 792d15224f2282268f4e6a91f6c619edddbc38f2 Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Thu, 19 Dec 2024 21:44:15 +0000 Subject: [PATCH 6/8] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 447288 -> 447288 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 100192 -> 100192 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 35909 -> 35909 bytes docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index d33c9f20808228662cabf018b2ce654fcb95d7b2..e3b8c17925a3e42249f50857ae4742d757407d7f 100644 GIT binary patch delta 9109 zcma)Cd0f;-vbTFK?pp+g8=@%JAgCyEd4Y0>S2TzM3LYaeL?DSdgv96uoLw-D*2`vH zakIuXZoGb3f8JsaH;K1NHZiV=?tWhKl9*L4QAG2qyJv>@{(Aq=Ro^ZAfSbJ&iQN}Q+=t{J2_5_R zB-Y6>7a5U6`Z?k(h9!YsIwWO?)njlnB-|zR#PB3=j{^prCdqcQ?ASok!xqK{8vD4& zN6y{j{SUB~Sfd!8nP`R#5d$+*PVo_$HJsw(Fl#x*M_|@*ipOo%bBf1nHgF099b$Ik z6pzj9%qbp}*@aWxoO0z<52xHX#p5!&Yqa=xK#7k_)rd%%4V{}qs^1=|7tu5uju;~a z$0KSsJe?vr9#ym9g{minj7QdN*pxC{a6GzZL&&+2g5wc38}3Gr5*&}R*^qxnay-&z zL&5C~A-gw8uxe4JV0oa;hPjs~3yufdY`FaQ9Kj6;?)lxhg5!g0HuOC=z4~o7+%%m$ zU^S*7a+hr~1sQmtEp0Ju)@a|*RxyRhSv?aH5|>w2Cnt*fs9D?AYVKs$O`)Wp4VZ#S zFAGdEk}ftTDV*G6PbNiUCSOQOGVma{v(uyfNH$xXJcj+4{KiEwbzC_MMl3D0lQVEdpIRZlJJ@nh3LU4 zQHVxShrG8KZ6N*?6*~sOdaJh1vb47CrM4LJ1Lge0C;}OWh&O9L=&Sf;JTEvOr9JbY zjjG7LWuul%CY0;?z)RbRL3wfNBPoErn~8z#%5+!SbG#_Hogx+%Hlq^Oowm9C%ZzHm zIUm-ZX;2dL(RJZj5~%bSbV?i=-$aMnqF$StX`4aunQarY@GS-bzL^Z?(#!#(zrjtyg51O6RLR z66xD$3yBvEMk@?p?<$QR+^aM$N_^8f>9Fi9%13PRgKv|mUWwauR@yuNwo4FLS4JpF zp>A+y2k};>ww#b~-$tX7xcN)T>|HaA#{=JfPn?yB?22>|zD35uZoM*0Nr$hbE@v{{ z_}XCz`+vYW%YE}7(#W(ej6!t-E)Kct3}e@}=R(bP+ak$n$3=piTqGBEoRFNd?X7z@ zYS&I&i9%!7&Y@5?j%t;YyFPmya)q&3IGE3OAD1}rNlcI&%Nrl|4?0%DeODAzx9(;;9=Q?}W7?d05IwiC1Edz`D(*NMaA0?l0HRSb+1||Hm zEkaagH_rJ(V-L|Qrq8e-d6=TogTIggG4%0P+?O)sjH?4&C6LX(9;j4Z!(u>vrRX{? zDQ+kh*s5cqxv_Erww#O1-2*Z&*KnG>)5_id@iBX0fv@rafEKR8p! zGGVQxrG~MbTe;vh57XLyYdztV544`fxt#c)Ba#(!=y4~Af=fKwoJb92f9uMHtMMdS ziRiWvPIZ#^y2+lJ) zVU1M=IPOh?whjUzVkJj6ijF|Ct|u5m^2FtE=UIOReN zkf$L&s^Us;E93%RMQW=%c_dgB^|0t{eX7_erpTdo$(LaMI9sqDJ;eu+P-r=>byfZR z$+yy87eG7*Ra8{L(nSjuV|A0%;!xrzRa$po-u#Hl8^+>HVo~f&Ck5OPSH3}Y%_0|sX2U`f3yE1o zuLe&dUx|+OQ;4hRIBFW%AiCmv4eXjhV$>gU$xflvK7%+5rDOTTNhqNw6POC9v+6Mu zGs43_B@>joQypX$VHrQa0?Rmc7CHWy&_&IfLoNvjdyd^Y6E(96F|nod{x7`F1L1Y^ z$rYh(AEY}B6jD!?(&DWie2N_7G8Kc-Iiga}EFxbDcwec6S=I`JdcK@I!*%$sq?T0R z+Tbx`#ks3l%Se}l1LVTgniaUz>-pN_SlI@?ZaLNX8@>vWV(C|%)Hj>#8v8bpM5%H6w`8BR-`7I$vVywM z3S_wY*=B;LpM(9B5!}kTC)_AOZ6Tp*#}*r;qQQ5+(tY<0a$2xz!<*!m6x?tF_xOX};LWWO}6?m(BD z&@?=fdcn0~>ZV@Ug{wsvyt|w4Ogw%sT!-q7J;X#=OqYefQz}?%k6hn|eZ&jTqaGi) zwv+g&>HBff2o{&JJ1jax0>M;*IWQf>CC3G7(0e=(P8A&@L4pF2HKBZM9u0t7yK(iL zJ&c7W^xisZ(^GL}E)zU-k>Ym- zSbi1{)pM8hu4=<)#EX{jmqDI)LiqnY5YVVnc;}|#$tSmMc<~=3 zSiCdwK`r8K70-@Aj(Z>E_&a2f;}4QSj=xL>IsQZ$BB>JW|j6K#o$Ugr))qp3j(CjQDHmp*Fr9CX?~nr5P#8>8uDY@(BCmNd%6YqIp_cmlc_oGqEvEpi>M&{AP@- z%HelNjQg)${*fmrL3H=c$9iW+ci@sHTok9#92w#+UN8Z0Dvd@<;2rQtrwOlLbY!#> z5+>3j?6+YePHrcho`_}H3C`(sIyQ6CX^}KME~jd11`T|Ce5ZgU&;Z0oO2zk~ED^iR z$)HIR_{gph#NT0_=(Qt*PKDd!@IK*^NheCJ3yw(r+5R8&(3nY`P_-#j#PL}suGoHX z$-=7Zhr%peLA|g(3nT7@kFqd5y<4+!arHt@Hda~}Y|N(ldhtEyHlO3s{6*wtE5|N~ zn1su#3#um3JZuh6q6Jd-0oR@AD?a4g=3Dr9BodJXm)wZEW460s$7EW7_O~a~8PeVW zJ94luFJC;W*G-{L5_N&3sdTJ(%fR_5)D=yxPNB20iJMAiVY6ZS>>LpE+XxMaFuR*ne#4>>d+xgTGUbRIG57l zlj(E{+;~OjZJU4|_;5NcL!~EX&=P}Cnl=)=_&&FN2Ck_dxH5yzlh*mLqX28%Qby3#+yE2-|BjxqlnIyP8)2-v#?slP+xHX_;A5%{Gie4S#*xnc}?mh&c=%D zf$G_`6q^gPaf9!HkU4ZdHuZC8sZ<8oyqNmIf9BZ7@p}mr%%$U{#x@DG&&B;)#LzB* zM|0@}y|{erk{!M`kB)$Cfi%FrX9(dt4&gMZ{jLPA&B70F$Boo^YasQ3lR?zgK9)Y1 z`6Oh7VAuXg zqAd$>7wm(M1z7ie;Pw>mWPMQb6wZf;{}2@C(jW*9$GrtFT3C!5 zLLY1^#$|=#V!9Zc!iBgo^ud;ev_vZZTPnMj;C4O$g(Y-_o?nB^Kg&2Sm!MYx+>t=W zBAjsnbW32{A{vL5pDv<{C4N_WoU|DCssXSr#)GySK3a_1Quo$UJixjks}wh`Zg{nn zj>qO?DV9VxJe1tHGMZr!KMVMbf>UF$4DXG^P1YW3@>v><*T1`GsYYILtbTAYj{1x1jx`iK z;%T^+--Y1iJ2V(?IVGJm0-JT6)QHXRJ82ZTtDfwn7oXMgs|;i>*F=b8-x|b{mKk;8 zW(NlRRIRVog`|sySus0Xpnd?yVDrLUB9_XV-p8J`znneT*9x} zGMCj*X>C6O4_uN)sae)GV`8?G>7U zG!d8IV6fsR+S2+pRkgO5{Rd_BHTV;DW$BWt^6FJ6g^1OB%xczZ%}_mt}01nYJm21S$Mrn!?Cf1Yh%RWZ4F~dM}46+RO^B8DF^J&;jIx`ozw~yT6-r@ zcj+U5;q67!W)_|}<7p6gdMv@4p9bj!4cQ^uVba_Pr@M$BbcAZ1B@oF0tjO?hFTqw+7Z|w=g%)G&Ty(# zqa(vu$G?WbSFIX1Z-<@y*8uD@J4zdbCM8k48?_-yyVP0FUu@O@D7!}jskvXpTD}mf z=MOjQFxGH4fSP-uGf(TUzn&P}sx1|jHp^hZ zZ#Tg>XLzGIt_=_u71wh7KM3hK63A3s`$8CF594MzPLxU28fX`V17Rc#T1vGp_E@3- zLO(zM!=6|HZXsIX-`#7iPrv~q^~2-WTV4I7E`M$) zKhmsT6_r($xY{uhrOWGAR^aprd?+*&(_y4fy&bD(Y2rkywZef-WGI;8^j_+_%k`&c z;wfZN^-a9-8an&X8|FfxUtFK>@ F@V|_jmp%Xh delta 9146 zcmb7KX57^EhK}UH%%>A zo7lt#mz(++SIlCQxJ8pTHP%GiUz^v&R8d62miNreb1$af-Y=YU{%4spGiSEva_r5Z zV{Zl>`fAKpt+{$-ZL39FR$sRk$?A2L6|EMB4Hb1|iP%2874D zsx_W=BOAiZW23g-rz1vK9C$Ku8*A%Y%@4TjC`=^qBL`e;J`x)gPP{)-?h-b(#wFIx zF(=szi9F(nqsU7F19U{v2#ecr2PNDq>_iVG@Q?!r92P0|un8d7Z5bWRbFy5 ze%^jRYaSOQx@RVu5hFy$%#?GyM`jJ@ct6Zq&hZ|Yb)4gQoAsRI`I-%!!$e1z9XQ8x zGdpsQCuMfxTo31*IoHQI7tZm#%&rxbsb5GD?#~K|cdp4W?FLG0lxZfp&uUD8 z=v)+1%P929)dEZ8}jcLeKk?d&l3f7tuiMXDnrz~d1Ns&xRDPwI&(^5-VNLnO2mAZoMofye7 zCaz+?PK{&@Kj>gfEb(T4pP0q`5y+fW%6w8H;l@^CWM5CpBK_D_Bs6r8AhtX$i}Z-5 zJJafAy*BU);tSFI8s|j4zR5XJuU5{9dhO(#sMm{}6ZQHP=S01J%Q;c6-JBEkN_atv zDs|(WsL~)&q`b9gZ2@f^N+`GWg6H=TgY~)TLs9`9uMh*+vUE!DE{hQT^^iIK7oOM;;xkpx0kESZS&oW#B>%4H2zqGN2)viB3jmS?D_s@u4( zqPBH;YjlB`Z<9M(3&gB&%4dR8J`X!u;Y=LyFDekD%~3wqobsXE(OSW`&sCN6jR(>cLIbcEMk0czm+KCE%)>?1Dz@cUw&`L8E(!PdDL0Q!5VC%uE zTf-q(P`R)5OwA1$reoueQpG3K+FAENNX)dyk9}5WfY>z}1IsM;wDxa0W7oiz1nS1V z+vI7boBt%keDcf>QXRG{2!;}V|JdG{*J%AFBmMM zASh0vE^uxy@w85FJ|*EtJA$n7uY4(mj*WgiANb~5;%J@BuF4Q$yJRlx&I^9lboiUJ zMzyWW=j>M=Q+Iqg;C)njroT-tkz^VU84 zdIjD8#vP&C|HeotO`%%r>HVKS4md+{7UuuOTPHaRyw}#Z4*c70awCO0vjhM1v|fGZ zI0vo2Js3bC%Y;i(NeO2;M{>bqAqHqa@)F^kmo??xPL*>!`bcuJx&n^%sG?Zke>`Vo z)$i%8dB?9ym{oU13Z(QTu0|dU+TAgoqSG2GLsxg=hQSqL2{*oP3oo{h*FNlfEWGuP z=LUzvTEG6-7E*277n-|>tF`~r3#wJeMuYX{1>7P;{5|^&Eapq!BbUEZfsAXxuU-tpMSFJhf-f|56TQ{+`RCG3)S=b*7d116arVW0sivQCZ9ugW#kKfGfVJ}aRS8)O zZ%}C_m)iPo`qh>TZ-m344C=sAUh%XZxphXh-~P`BD);ig?y9D|4=vyRS0Aj0ZaY$_ z+KMxlmKw@(?&LyE8=tW|+X?5qpye!%^5p-#Cq*%~ZubH)n-26jl^VkSbUzm|j;fn) z_=9S~B`=n-5^JSfQ6;{|ZtnTFl&nwoVj9AV@2JoA87Vl$Vz>-E0LKjG!nIfuW!*Jc zM>yvRCFh9)?03gKJnbh8D6lqn5&@wnv`%oSkLaMuk$5W83Hj*1)b7eG4Z-2(K8i_4 za2SPqrjbPK9w4slgSR}D?S{ww#z4ynt%HIQ|2iCO44iQy2FTYCFU9IiK2$X~$6^+r zxnP{zo4k}yTy1d_Oi!yeg{?+_hc_`OH$2H#k2@mhsyy;0ST!Q4S=+Inv4JE6nonw- z6(3*njnpVwe}ZS7-AF+rEOyZ;?1`(&E)O9iRjcN`#6j5^MsRTmtF{Om3%u*Fhf-JN zA5r9@+Cb+hf+Zyy;Q8*Z&>EoqxK^k1MU%g%HrUy*lttgKwp~t^v6gPJ$-rPNIXY3C zGd_k9BgARPZwUPS;`aki`Q-`EF>{zSkT(c%dFWT}#E}Jrz4v`J95j)&tocrql9oU= z5#&5s`ZWhQ@GLQc$6Oqqb{#b+562TlD3w!51dn4hQ@%^a&I`_CqKW}>Nl@~V&V&7a zx~KAL8u>tYK}!vZvWIhkLz9WK@u1(t*{04VCx5PWQnKcg&m@MqBtTXHPG`wN6;lxNkZss{>Iwq| z)SV?Ydn!kkk`pp1Oh7-JigIo_!J~=i*ioV?Sms&>EEVusF1n#7XYnFG+{=$2-*> z$>E{&FTwHi;|b+cM({$!gW%=~N;3&jI(Mm%v^WZ+^D&2aUL$8^B)ryqFF8CE@$#c) z0+@+A)D%&7xVDtKDSv7spGdQoy||5u6yMxOA|-eI4RTPrR(E3iJJB^Pf_lL9BI=@C z+RxVwABm2)2;L5ngUcszo4a{{m?-H7&kk~x-Aj*B34*4I;o|qn z#1)nwB>`Y6#vwNy;j0c6O5nRBiVCjqC`^IkH|yGt@<;$D>x{= zNCK4SKOtA70?+I3RO5>($FE8O@XZyygR=QDU)wy?k}IT3a14)PSaTi^*9)KNot3Sh z<7zGDuY-K~sLyv0@3nj?o(TD&;)Rg!D4uEgM)5|=*N6vNzD~Rs^8LhfA)km{HU9~* z=_1JYTnyF&-~WY7E#OZQ+?xNN?*??L5Yf4-c=pLn8=m_sF^ab)K8Z!XE#fgUEU8a> zRmYzv!#e&>8J75iWmw`bmtl!NVTL7s#tch6bAKlC?6{H}wTLkomejYq)YV&D*Hl)o zEWxK~-Np)hPTB0tKPqd#B&#U7uUxps&wszsxE>B&C)=rm96s})lJ+%eawc8Mx!dG8 zCEd!lJ0#DUJc0|4$h%}f+15+S3F%e-{Uh;qA`hWYL(@s0lB%Ws-egEg@~6k0NWXG7 zf<|jekK#X??$^xWv)?GjuThLwqnP?eG4qXL;v2=hH;QR*6tmtaI@2g7rBQUZQOtNF zPPp4RT1@W4`f)TF$?wL|sRrJJxob3JO{0F`c%4MMna7SDF+#0}?$J=1hKj-%Iu<>i zjiKYo1L%ms+#kT57@DgW9}@`tGy&Jyyjc2#1himELSR=c9gT*kVrd4Ffmk{N$<#QS zL&1>QZf0l zfLuG&;65C*eLT&?FgM53sYs#{X_j=##RrazW`7wv8{|k1LPsJFz##mPh~qX0V@j;7$tp6o0V46S;s7#$8T6KzE^Qy*31)D!Ok(w~@&53jl64N9)%^=jTN6V@lb&o{5J|6P7e1hUc zb6#S+YM(Y zV_9~CV>+FQWPUm=l#VCmQ0>m30YC5G8NgH8AMr8L@O>za$0l#| z?aaWs?go7(7LVxZC3~8_?9gpJG-Of-G-%8eNqnA(%k>dBWnsxZf`TkuJ_E2l3)3Bd zkFs!X26kuT${K*2Y%H^0*pW?V>BTpmyL^Dh^9PZKT8q69J_XlSFH}yU`ACjUp|hoJ zKewIiEq)qM2V2A#60?vX-ZS>0?uETm>1_1BJC){1e*^5z!M6D81-4A14ia^Ogy}R{ zyjDXVRs5aVVw3Co}0ZxcP$4QyqXl_;4mIMWZM3 zXt6;U%@_k7e3NU-!v)m`SM%sX={*bf&c*_-E}&z@T^}~)Q*Wu7BUSDBxOhd7c@h}N z$GPf*#94Ha#Ft3?_p|5}2`rO9>>L`;ulXKmojsctA^8mwX;Ut3u;F-qiT9D{8ap~w z0%i$ZETGdRu)!WX6XR`|V;hEAJ01@ng`&;yLb^b9fAbh{Psi^6!jnE>+vobAvyje{ zLAR>TbLnIWykG}R;_g1|bapPCFKu3yHu3YYBKx3f9xXv~X&&zGeGojKE=E#6pO#2t zfLB&fANbFFTR(m+f!Pb_L}{@{0__WM^A^M1E`gy1bdp}&Jod{5-(5&Y!=3=@Z`&}0 z@@>0vhO~Z10@vr_$GDS0)NywJ^@7uZ)Y;aTA(*oW_thczpGCA(`dDR?UW;jp1m2TD zaQ};fmlsnPiJr8hCUn2Kn9h{=`*u7E)-JJGez`yp^yird%_6fec` z5cwa4qFfpX#xUGi@Gd?Qm3Noo{0QY2GOuMtxFHO|o+4aUs4k)_kQ6M#jbR9OEu+QK z_@C0)xfu8IekdrWYxVpNWd1?saitie3gDguGM3|r3*dnS_AI9{==te#xt!eNi34Q%}KxVaT)!?I=t3Ac^;P-zX(|Z*tY2b4z1!cY>7Z2 zlA7zX-v$d(?DTj$}(04MOtk zE*eQ5D5tyV<)^g#9s}8HG~wdRw*<0;)j>LOt%JiAnpmlCsL+hYgFeBm8D%qTs|v!4 zKRjdBM4_h5tO-VP%B+b*a?7jPM9zQelu8vHH0 zqGV-dS=D;fLih&WXC-Tc=KBb7nX!bzE6Frm%Wpam+pO`!n@d%TCNPx)7H=LYkN>QU zBT`aZS5{Y#jR=)Uz^>X(+#6a#v~CEWvBSO`-W{&hNvjZH)iH^>${79(uP&Navxvk6cZ0alV+r2+G*Bma z$PU)}N%tT)d!P6~XNcBO0uda*>gyDWGmWmJBKyO|5L%-3#Jp=aB-us1VL9nF&e~M#}W*$IFMjmu-sQl0d}7=C$y2$ zCm7DU5Th1nTtxB_A6{Lms>>@>Pb1)`8Y2!JUTjWk{l$I7xeWjPK?aThG99-7p^Ubr zajOi+%cN)xw3ou+P!bBwC0Zw2E|GxxNShSIr9elX08casenqLR4?aLf5g*Yu_R`1p zkyS6-F+x|lR%E99a7z2_IdS}1M!~HlGFm4dc`*2b&Zy<5B3ybw7by)wp)8GrQQQEZ zd`TBEj%zJ$)2g4Us;n+6pRbm`KwWHT*-HGk5p+2HlFo?7^Yxc>fu6#`VRl*7y44l@ ze>nO;ze5+{D_~a}9LHmFUB(VwuuLTDMO}nUBna?3nTm$LsPo3iCwJ(=@#*{Z4&ACk z13$uQElTBNk)PxM(?4#vqpb*4jx9;iat^tQuP*?dz3^eCl&qbKrKI2pzw;`7tfIN ztNLI)2P}b5Rzi(ZS#w42Be`E)(FaKGupPUMSeWn;TgT$s`E}JB%hpuH)z`?85DlFC aOmFn%T)o9%4KBVo{?9jct@ZUQ~ delta 132 zcmaFR$M&F)tziozpEa+ExrwQ)gX!oDrVal!dCg2r%uEgS%*`z)USj)Z%1C72X`LzlQ@=XtIWdKblC3 Ng$t_4=BpM3!2lQl93B7w delta 106 zcmX>)gX!oDrVal!c}>hsOpQ(S3=NG;EiEU<>)USj)Z%1C72X`LzlQ@=XtIWdKblC3 Ng$t_4=BpM3!2l4w8~XqN diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 5b27d3588..6611db4aa 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"DRS Download Tools": [[10, null]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Gen3 Auth Helper": [[0, null]], "Gen3 File Class": [[1, null]], "Gen3 Index Class": [[3, null]], "Gen3 Jobs Class": [[4, null]], "Gen3 Metadata Class": [[5, null]], "Gen3 Object Class": [[6, null]], "Gen3 Query Class": [[7, null]], "Gen3 Submission Class": [[8, null]], "Gen3 Tools": [[9, null]], "Gen3 Workspace Storage": [[13, null]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Indexing Tools": [[11, null]], "Indices and tables": [[2, "indices-and-tables"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Metadata Tools": [[12, null]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, null]]}, "docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "indexentries": {"_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager", false]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods", false]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS", false]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create", false]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases", false]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record", false]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias", false]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases", false]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest", false]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get", false]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases", false]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record", false]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum", false]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page", false]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params", false]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest", false]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls", false]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd", false]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait", false]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update", false]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases", false]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record", false]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest", false]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider", false]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ", false]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create", false]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens", false]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children", false]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID", false]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url", false]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy", false]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create", false]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases", false]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank", false]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path", false]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job", false]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version", false]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list", false]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program", false]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project", false]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record", false]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time", false]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl", false]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR", false]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete", false]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias", false]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases", false]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids", false]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file", false]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations", false]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path", false]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node", false]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes", false]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object", false]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program", false]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project", false]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record", false]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record", false]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records", false]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download", false]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download", false]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download", false]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest", false]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single", false]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url", false]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable", false]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager", false]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus", false]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time", false]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint", false]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node", false]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record", false]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name", false]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name", false]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size", false]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size", false]], "gen3.tools": [[9, "module-gen3.tools", false]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download", false]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest", false]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest", false]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest", false]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest", false]], "gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth", false]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File", false]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index", false]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs", false]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata", false]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object", false]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query", false]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission", false]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage", false]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get", false]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get", false]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token", false]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts", false]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases", false]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records", false]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all", false]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node", false]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token", false]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema", false]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix", false]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths", false]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version", false]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output", false]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url", false]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs", false]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary", false]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest", false]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects", false]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record", false]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc", false]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records", false]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page", false]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats", false]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status", false]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls", false]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids", false]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version", false]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version", false]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version", false]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions", false]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params", false]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query", false]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID", false]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT", false]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT", false]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname", false]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest", false]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE", false]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy", false]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy", false]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy", false]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest", false]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object", false]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest", false]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs", false]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load", false]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest", false]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls", false]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path", false]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest", false]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS", false]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5", false]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum", false]], "module": [[9, "module-gen3.tools", false], [10, "module-gen3.tools.download.drs_download", false], [11, "module-gen3.tools.indexing.download_manifest", false], [11, "module-gen3.tools.indexing.index_manifest", false], [11, "module-gen3.tools.indexing.verify_manifest", false], [12, "module-gen3.tools.metadata.ingest_manifest", false]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id", false]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id", false]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type", false]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project", false]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint", false]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID", false]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query", false]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query", false]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query", false]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls", false]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download", false]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token", false]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects", false]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm", false]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path", false]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE", false]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time", false]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status", false]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file", false]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record", false]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl", false]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER", false]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update", false]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases", false]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank", false]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record", false]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time", false]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload", false]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file", false]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid", false]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url", false]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS", false]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access", false]]}, "objects": {"gen3": [[9, 3, 0, "-", "tools"]], "gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "terms": {"": [1, 4, 8, 10, 11], "0": [5, 7, 8, 11], "0084667": [], "013444": [], "0248635": [], "0791543": [], "0a80fada010c": 11, "0a80fada096c": 11, "0a80fada097c": 11, "0a80fada098c": 11, "0a80fada099c": 11, "1": [4, 5, 8, 10, 11], "10": [5, 7, 11], "100": [8, 11], "1125998": 12, "11e9": 11, "120715": [], "1329331": [], "1468077": [], "165499": [], "1724170637": [], "1726237606": [], "1726237962": [], "1728059996": [], "1728059997": [], "1728061593": [], "1728061594": [], "1729113765": [], "1729629354": [], "1729629627": [], "1732401632": [], "1732401820": [], "1732401821": [], "1733254905": [], "1733263055": [], "1733431971": [], "1733932163": [], "1734031012": [], "1734031066": [], "1734031067": [], "1734474422": [], "1734474533": 11, "1734474534": 12, "2": [5, 11], "2023835": [], "2310405": [], "24": [11, 12], "244287": [], "2511687": [], "255e396f": 11, "256328": [], "284675": [], "3": [5, 11], "30": 8, "313695": [], "33": 5, "333": 5, "3410327": [], "343434344": 11, "363455714": 11, "3686683": [], "3697538": [], "3759608": [], "4": [5, 11], "4401503": [], "450c": 11, "4714": 8, "473d83400bc1bc9dc635e334fadd433c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "5": [0, 5], "50": 7, "5117643": [], "518752": [], "543434443": 11, "5833347": [], "6129959": [], "6138082": [], "6146133": [], "653854": [], "6f90": 8, "7d3d8d2083b4": 11, "8366766": [], "8420": 8, "848236": [], "8493605": [], "8535511": [], "863009": 11, "8712757": [], "9082778": [], "93d9af72": 11, "998105": [], "9a07": 11, "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "AND": 5, "Be": 1, "But": 5, "By": 11, "For": [1, 5, 6, 7, 8, 9, 11], "IF": 11, "If": [0, 1, 7, 11, 12], "In": 10, "It": 10, "NOT": 12, "OR": 5, "One": 7, "Such": 9, "THE": 11, "That": 3, "The": [0, 1, 2, 3, 5, 8, 10, 11], "There": 11, "These": 9, "To": 11, "Will": [4, 7, 10], "_get_acl_from_row": 11, "_get_authz_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_for_row": 12, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "_guid_typ": 12, "_manag": [2, 9, 10], "_query_for_associated_indexd_record_guid": 12, "_ssl": [3, 4, 5], "a5c6": 11, "ab167e49d25b488939b1ede42752458b": 3, "about": [2, 3], "abov": 11, "access": [0, 1, 3, 7, 10], "access_method": [2, 9, 10], "access_token": 0, "accesstoken": 0, "acl": [2, 3, 9, 11], "across": 11, "act": 0, "action": [9, 11], "actual": 11, "ad": 3, "add": [3, 5], "addit": [3, 5, 10, 11], "admin": [5, 11], "admin_endpoint_suffix": 5, "after": 10, "against": [3, 7, 8, 11, 12], "algorithm": 3, "alia": [3, 5], "alias": 5, "aliv": 7, "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "allow": [0, 6, 8, 10, 11, 12], "allowed_data_upload_bucket": 1, "along": 2, "alreadi": 9, "also": 1, "altern": [5, 11], "alwai": 5, "ammount": 12, "amount": [1, 9], "an": [0, 3, 6, 8, 10, 11], "ani": [0, 5, 10, 11], "anoth": 5, "api": [0, 5, 8, 11], "api_kei": 11, "appli": 7, "appropri": 13, "ar": [5, 7, 8, 9, 10, 11], "arbitrari": 0, "argument": [0, 13], "arrai": 8, "asc": 7, "assign": 9, "assist": 10, "associ": [3, 5], "assum": 11, "async": [3, 4, 5, 9, 11, 12], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_create_record": [2, 3], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_download_object_manifest": [2, 9, 11], "async_get": [2, 5], "async_get_alias": [2, 5], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_url": [2, 3], "async_query_urls_from_indexd": [2, 9, 12], "async_run_job_and_wait": [2, 4], "async_upd": [2, 5], "async_update_alias": [2, 5], "async_update_record": [2, 3], "async_verify_object_manifest": [2, 9, 11], "asynchron": [3, 4, 5], "asyncio": [11, 12], "asyncron": 5, "attach": [3, 5], "attempt": 11, "attribut": [10, 11], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "authbas": 0, "authent": 0, "author": 1, "authz": [0, 1, 2, 3, 9, 10, 11], "auto": [0, 2], "automat": 0, "avail": [1, 2, 10, 11], "az": 1, "b": [5, 11], "b0f1": 11, "bar": 10, "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "baseid": 3, "basic": [3, 11, 12], "batch_creat": [2, 5], "batch_siz": 8, "behalf": 0, "behavior": 11, "being": 3, "belong": 8, "below": 11, "blank": 3, "blob": [5, 7], "bodi": 3, "bool": [4, 5, 8, 10, 11, 12], "boolean": 3, "bownload": 10, "broad": 9, "broken": 9, "bucket": [1, 6], "bundl": 10, "byte": 10, "c": [5, 11], "cach": 10, "cache_hosts_wts_token": [2, 9, 10], "call": [10, 13], "can": [0, 3, 4, 8, 11, 12], "capabl": 9, "case": [0, 10], "categori": 9, "ccle": 8, "ccle_one_record": 8, "ccle_sample_nod": 8, "cdi": 7, "chang": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "child": 10, "children": [2, 9, 10], "chunk_siz": 8, "class": [0, 2, 10, 11, 13], "cli": 10, "client": [0, 3], "client_credenti": 0, "client_id": 0, "client_scop": 0, "client_secret": 0, "code": [2, 8], "column": [11, 12], "column_to_use_as_guid": [2, 9, 12], "columna": 11, "columnb": 11, "columnc": 11, "com": 7, "comma": 11, "command": [10, 11], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "commons_url": [2, 9, 10, 11, 12], "complet": [4, 11], "complex": 7, "concat": 11, "concurr": [11, 12], "configur": 1, "connect": 12, "consist": 3, "constructor": 0, "contain": [0, 2, 5, 8, 9, 10, 11, 12], "content": [3, 13], "content_created_d": 3, "content_updated_d": 3, "continu": 10, "control": 3, "conveni": 10, "copi": [2, 13], "coroutin": 11, "correspond": 3, "count": 3, "crdc": 0, "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_blank": [2, 3], "create_index_key_path": [2, 5], "create_job": [2, 4], "create_new_vers": [2, 3], "create_object_list": [2, 9, 10], "create_program": [2, 8], "create_project": [2, 8], "create_record": [2, 3], "created_tim": [2, 9, 10], "creation": [3, 11], "cred": 3, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "csv": [8, 11, 12], "curl": [0, 2], "current": [6, 8, 10], "current_dir": [2, 9, 11], "custom": 11, "d": 5, "d70b41b9": 8, "data": [0, 1, 3, 5, 7, 8, 10, 11], "data_spreadsheet": 8, "data_typ": 7, "data_upload_bucket": 1, "dataa": 11, "datab": 11, "databas": 5, "datacommon": 0, "datafil": 10, "datamanag": 10, "date": 3, "datetim": [1, 3, 10], "dbgap": 12, "dcf": 8, "def": 11, "default": [0, 1, 3, 7, 8, 11, 12], "defin": [5, 8, 10], "delai": 4, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_all_guid": [2, 9, 11], "delete_fil": [1, 2], "delete_file_loc": [1, 2, 6], "delete_index_key_path": [2, 5], "delete_nod": [2, 8], "delete_object": [2, 6], "delete_program": [2, 8], "delete_project": [2, 8], "delete_record": [2, 3, 8], "delete_unpacked_packag": 10, "delimet": [11, 12], "delimit": 11, "demograph": 8, "deprec": 1, "descript": [3, 5], "desir": 11, "dest_path": 13, "dest_urlstr": 13, "dest_w": 13, "dest_wskei": 13, "detail": [2, 7, 10], "determin": [10, 11, 12], "dev": 11, "dict": [3, 4, 5, 10, 11, 12], "dictionari": [3, 4, 5, 7, 8], "did": 3, "differ": 5, "direct": 0, "directori": [10, 11], "disabl": 10, "discoveri": 10, "disk": 13, "dispatch": 4, "dist_resolut": 3, "distribut": 3, "doc": [7, 10], "docstr": 2, "document": [1, 3], "doe": [0, 12], "domain": [11, 12], "done": 4, "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "download_files_in_drs_manifest": [2, 9, 10], "download_list": 10, "download_manifest": 11, "download_singl": [1, 2], "download_url": [2, 13], "downloadmanag": [2, 9, 10], "downloadstatu": [2, 9, 10], "dr": [2, 9], "drs_download": 10, "drs_hostnam": 10, "drsdownload": 10, "drsobjecttyp": 10, "e": [5, 10], "e043ab8b77b9": 8, "each": [3, 8, 10, 11], "effici": 9, "eg": 3, "either": 8, "elasticsearch": 7, "els": [0, 12], "elsewher": 12, "empti": [8, 11], "enabl": 11, "end": [5, 10], "end_tim": [2, 9, 10], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 10, 13], "entir": 8, "entri": [3, 11], "env": 0, "environ": 0, "equal": 7, "error": [10, 11, 12], "error_nam": 11, "etc": 8, "even": 11, "everi": [9, 11], "everyth": 11, "ex": [0, 11, 12], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "exclud": 3, "execut": [7, 8, 11], "exist": [1, 3, 5, 6, 9, 12], "expect": [5, 9, 11], "experi": 8, "expir": [0, 1], "expires_in": 1, "export": [8, 10], "export_nod": [2, 8], "export_record": [2, 8], "f1f8": 11, "factori": 10, "fail": [8, 10], "fals": [3, 5, 6, 10, 11], "featur": [1, 6], "fenc": [0, 1], "fetch": 0, "field": [3, 5, 7, 11, 12], "fieldnam": 11, "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "file_nam": [1, 2, 3, 9, 10, 11], "file_s": [2, 9, 10, 11], "file_st": 3, "fileformat": 8, "filenam": [0, 8, 10, 11, 12], "files": 10, "fill": 12, "filter": [5, 7], "filter_object": 7, "first": [7, 8], "flag": 11, "folder": 11, "follow": [0, 11], "forc": 11, "force_metadata_columns_even_if_empti": 11, "form": 13, "format": [3, 5, 8, 11], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "func_to_parse_row": [11, 12], "function": [2, 3, 4, 5, 9, 10, 11, 12], "g": 10, "gen3": [10, 11, 12], "gen3_api_kei": 0, "gen3_oidc_client_creds_secret": 0, "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "gen3fil": [1, 2], "gen3index": [2, 3], "gen3job": [2, 4, 10], "gen3metadata": [2, 5], "gen3object": [2, 6], "gen3queri": [2, 7], "gen3submiss": [2, 8], "gen3wsstorag": [2, 13], "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "get_access_token": [0, 2], "get_access_token_from_wt": [0, 2], "get_alias": [2, 5], "get_all_record": [2, 3], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_fresh_token": [2, 9, 10], "get_graphql_schema": [2, 8], "get_guid_from_fil": 12, "get_guids_prefix": [2, 3], "get_index_key_path": [2, 5], "get_latest_vers": [2, 3], "get_output": [2, 4], "get_presigned_url": [1, 2], "get_program": [2, 8], "get_project": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_statu": [2, 4], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "giangb": 11, "github": [2, 7], "give": 1, "given": [0, 3, 4, 5, 8, 10, 12, 13], "global": [4, 5], "good": 3, "grant": 0, "graph": 8, "graphql": [7, 8], "graphql_queri": [2, 7], "group": 3, "guid": [1, 2, 3, 5, 6, 9, 11, 12], "guid_exampl": 11, "guid_for_row": 12, "guid_from_fil": 12, "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "guppi": 7, "ha": [0, 11], "handl": [3, 10], "hardcod": 0, "has_vers": 3, "hash": [3, 11], "hash_typ": 3, "have": [5, 11], "header": 11, "healthi": [3, 4, 5], "help": 11, "helper": 2, "hit": 11, "host": 10, "hostnam": [2, 9, 10], "how": [8, 11], "howto": 10, "http": [0, 7, 11, 12], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "id": [0, 1, 3, 5, 10, 11], "idea": 3, "identifi": [3, 5, 9, 11], "idp": 0, "illustr": 11, "immut": 3, "implement": 0, "implic": 11, "import": 11, "includ": [0, 3], "indent": 10, "index": [0, 2, 5, 9], "index_manifest": 11, "index_object_manifest": [2, 9, 11], "indexd": [1, 3, 6, 10, 11, 12], "indexd_field": [11, 12], "indexd_record_page_s": [2, 9, 11], "indexed_file_object_guid": 12, "indic": [0, 11], "infil": 10, "info": [3, 11], "inform": [2, 3, 10], "ingest": [2, 9], "ingest_manifest": 12, "initi": [0, 10], "input": [4, 10, 11], "input_manifest": 11, "instal": [0, 2, 11], "instanc": [1, 3, 6, 7, 8, 9, 10], "instead": [1, 7, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "integ": [1, 3, 8], "intend": 0, "interact": [1, 3, 4, 5, 6, 8, 13], "interest": 10, "interpret": 0, "introspect": 8, "involv": 9, "is_healthi": [2, 3, 4, 5], "is_indexed_file_object": 12, "isn": 1, "issu": 0, "its": [1, 3], "job": 2, "job_id": 4, "job_input": 4, "job_nam": 4, "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "just": [5, 11, 12], "jwt": 0, "kei": [0, 3, 5, 13], "know": 11, "known": 10, "kwarg": [3, 4, 5], "l": [2, 13], "larg": 9, "last": 10, "latest": 3, "least": 3, "level": 6, "librari": 11, "like": [3, 5, 9, 11, 12], "limit": [1, 3, 5, 12], "linear": 4, "linux": 10, "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "list_job": [2, 4], "live": [11, 12], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "local": [0, 13], "locat": [1, 6], "lock": 12, "log": [8, 10, 11, 12], "logic": [5, 12], "loop": 11, "ls_path": [2, 13], "maco": 11, "made": 3, "mai": [0, 9, 11], "main": 10, "make": [9, 11], "manag": [1, 5, 10], "mani": [8, 11], "manifest": [2, 8, 9, 10, 11, 12], "manifest_1": 10, "manifest_fil": [11, 12], "manifest_file_delimit": [11, 12], "manifest_row_pars": [11, 12], "map": [0, 11], "mark": 8, "master": 7, "match": [3, 5, 12], "max": 5, "max_concurrent_request": [2, 9, 11, 12], "max_presigned_url_ttl": 1, "max_tri": 8, "maximum": [11, 12], "md": [5, 7, 10, 12], "md5": [2, 3, 9, 11], "md5_hash": 11, "md5sum": [2, 9, 10], "mean": 8, "mechan": 3, "merg": 5, "metadata": [2, 3, 6, 9, 11], "metadata_list": 5, "metadata_sourc": 12, "metadata_typ": 12, "metdata": 12, "method": [1, 7, 10], "minimum": 10, "minut": 0, "mode": 7, "modul": [2, 10, 11], "more": [2, 5, 7, 9, 10], "most": 9, "mostli": 2, "multipl": [8, 11], "must": [1, 5], "my_common": 10, "my_credenti": 10, "my_field": 7, "my_index": 7, "my_program": 7, "my_project": 7, "name": [3, 4, 8, 10, 11, 12, 13], "namespac": [0, 12], "necessari": [3, 5], "need": [3, 7, 10, 11], "nest": 5, "net": 11, "never": 0, "new": [0, 3], "node": 8, "node_nam": 8, "node_typ": 8, "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "note": [0, 3, 11, 12], "noth": [3, 6], "now": [1, 8], "num": 5, "num_process": 11, "num_total_fil": 11, "number": [3, 7, 8, 11, 12], "o": 0, "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "object_id": [1, 2, 9, 10], "object_list": 10, "object_typ": [2, 9, 10], "objectid": 10, "obtain": [0, 10], "occur": 10, "off": 5, "offset": [5, 7], "oidc": 0, "old": 3, "one": [3, 5, 10, 11], "onli": [3, 5, 7, 8, 10, 11], "open": [8, 10, 11], "open_project": [2, 8], "openid": 0, "opt": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "order": [0, 8], "ordered_node_list": 8, "org": 10, "otherwis": 10, "output": [4, 5, 11, 12], "output_dir": 10, "output_filenam": [11, 12], "overrid": [0, 11, 12], "overwrit": 5, "own": 0, "packag": 10, "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "pagin": 3, "parallel": 11, "param": [3, 5, 8, 10], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "pars": [10, 11, 12, 13], "parser": [11, 12], "particular": 0, "pass": [0, 7, 8, 10], "password": [11, 12], "path": [0, 1, 5, 10, 11, 13], "path_to_manifest": 11, "pattern": [3, 12], "pdcdatastor": 11, "pend": 10, "per": [11, 12], "peregrin": 8, "permiss": 10, "persist": 9, "phs0001": 11, "phs0002": 11, "pick": 1, "pla": 11, "place": 11, "planx": 11, "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "popul": [10, 12], "posit": [1, 7], "possibl": 10, "post": [0, 11], "pprint": [2, 9, 10], "prefix": 3, "presign": 1, "pretti": 10, "prev_guid": [2, 9, 11], "previou": [3, 11], "previous": 4, "print": [8, 10], "process": 11, "processed_fil": 11, "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "program": [8, 11], "progress": [8, 10], "project": [8, 11], "project_id": [7, 8], "protocol": 1, "provid": [0, 1, 3, 5, 7, 8, 12], "public": [3, 5], "put": 0, "py": 11, "python": [2, 9, 11], "python3": 11, "python_subprocess_command": 11, "queri": [1, 2, 3, 5, 8, 11, 12], "query_str": 7, "query_txt": [7, 8], "query_url": [2, 3], "quickstart": 2, "rather": 0, "raw": [7, 11], "raw_data_download": [2, 7], "rbac": 3, "read": [3, 5, 11], "readm": 2, "reason": 10, "record": [1, 3, 5, 7, 8, 11, 12], "refresh": [0, 10], "refresh_access_token": [0, 2], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "regist": 8, "regular": 7, "relat": 9, "remov": [1, 6, 11, 13], "replac": 11, "replace_url": 11, "repo": 2, "repres": [3, 5, 10], "represent": [1, 3], "request": [0, 1, 3, 5, 8, 11, 12], "requir": 10, "resolv": 10, "resolve_object": [2, 9, 10], "respect": 7, "respons": [0, 1, 3, 4, 5], "result": [1, 8, 10, 11], "retri": 8, "retriev": [1, 8, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "return_full_metadata": 5, "rev": 3, "revers": 8, "revis": 3, "right": 1, "rm": [2, 13], "rm_path": [2, 13], "root": [11, 12], "row": [7, 8, 11, 12], "row_offset": 8, "rtype": 3, "run": [8, 11], "s3": [1, 10, 11], "safe": 11, "same": [5, 11, 13], "sampl": [8, 10], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "save": 10, "save_directori": 10, "schema": 8, "scope": [0, 1], "screen": 8, "script": 2, "search": [0, 2, 3], "second": [1, 4], "secret": 0, "see": [7, 10, 11], "self": 10, "semaphon": 12, "semaphor": 12, "separ": [0, 11], "server": 10, "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "service_loc": [3, 4, 5], "session": 11, "set": [0, 1, 5, 10], "setup": 2, "sheepdog": 8, "should": [0, 8, 11], "show": 10, "show_progress": 10, "shown": 11, "sign": 1, "signpost": 3, "similar": 10, "simpl": 3, "simpli": 11, "sinc": 3, "singl": [1, 5, 8], "size": [2, 3, 9, 10, 11], "skip": 8, "sleep": 4, "so": 11, "some": [0, 2], "someth": 11, "sort": 7, "sort_field": 7, "sort_object": 7, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "space": [0, 11], "specif": [5, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "spreadsheet": 8, "src_path": 13, "src_urlstr": 13, "src_w": 13, "src_wskei": 13, "ssl": [3, 4, 5], "start": [4, 7, 8, 10], "start_tim": [2, 9, 10], "static": 10, "statu": [2, 4, 9, 10], "storag": [1, 2, 6], "store": [1, 3, 10], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "string": [0, 3, 5, 11, 13], "strip": 11, "sub": 8, "subject": [7, 8], "submiss": 2, "submit": [8, 11], "submit_additional_metadata_column": 11, "submit_fil": [2, 8], "submit_record": [2, 8], "submitter_id": 7, "success": 10, "successfulli": 10, "suffici": 3, "suppli": 3, "support": [0, 1, 5, 8, 11], "sure": 1, "synchron": 11, "syntax": 7, "system": [6, 7, 8, 9], "t": [1, 5, 11], "tab": 11, "task": 9, "temporari": 11, "test": 11, "test1": 11, "test2": 11, "test3": 11, "test4": 11, "test5": 11, "text": [1, 7, 8], "than": [0, 5], "thei": [0, 10], "them": [10, 11], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "those": 11, "thread": 11, "thread_num": 11, "threadcontrol": [2, 9, 11], "through": [8, 11], "tier": 7, "time": [1, 3, 8, 10, 11], "timestamp": 10, "tmp_folder": [2, 9, 11], "token": [0, 10], "tool": 2, "total": 11, "treat": [1, 5], "tree": 10, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "try": 0, "tsv": [8, 11, 12], "tupl": [0, 3, 11, 12], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "typic": 10, "uc": 7, "unaccess": 7, "under": [0, 8, 13], "uniqu": [1, 5], "unknown": 10, "unpack": 10, "unpack_packag": 10, "until": [4, 10], "up": [5, 9], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "update_blank": [2, 3], "update_record": [2, 3], "updated_tim": [2, 9, 10], "upload": [1, 2, 3, 8, 13], "upload_fil": [1, 2], "upload_file_to_guid": [1, 2], "upload_url": [2, 13], "url": [1, 2, 3, 9, 10, 11, 12, 13], "urls_metadata": 3, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "usag": 11, "use_agg_md": 5, "user": [0, 10, 12], "user_access": [2, 9, 10], "usual": 12, "utcnow": 1, "util": 9, "uuid": [1, 3, 8], "uuid1": 8, "uuid2": 8, "valid": [3, 7], "valu": [0, 1, 3, 5, 7, 10, 11], "value_from_indexd": 11, "value_from_manifest": 11, "variabl": [0, 7, 8], "variou": 2, "verbos": [7, 8], "verif": 11, "verifi": [2, 9], "verify_manifest": 11, "verify_object_manifest": 11, "version": [3, 4, 5], "vital_statu": 7, "w": 13, "wa": 0, "wai": 10, "wait": 4, "want": [0, 3, 8], "warn": 11, "we": [3, 11], "web": 0, "what": 5, "when": [0, 3, 7, 10, 12], "where": [0, 3, 5, 11, 12], "whether": [3, 4, 5, 8, 11, 12], "which": [8, 10], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "whose": 5, "within": [0, 2, 9], "without": [3, 5], "won": 5, "work": [0, 10], "workaround": 11, "worksheet": 8, "workspac": [0, 2], "wrapper": 10, "write": 11, "ws_urlstr": 13, "wskei": 13, "wss": 13, "wt": [0, 10], "x": 11, "xlsx": 8, "you": [0, 3, 8, 11], "your": 0}, "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "titleterms": {"": 2, "auth": 0, "class": [1, 3, 4, 5, 6, 7, 8], "document": 2, "download": [10, 11], "dr": 10, "file": 1, "gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "helper": 0, "index": [3, 11], "indic": 2, "ingest": 12, "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "sdk": 2, "storag": 13, "submiss": 8, "tabl": 2, "tool": [9, 10, 11, 12], "verifi": 11, "welcom": 2, "workspac": 13}}) \ No newline at end of file +Search.setIndex({"alltitles": {"DRS Download Tools": [[10, null]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Gen3 Auth Helper": [[0, null]], "Gen3 File Class": [[1, null]], "Gen3 Index Class": [[3, null]], "Gen3 Jobs Class": [[4, null]], "Gen3 Metadata Class": [[5, null]], "Gen3 Object Class": [[6, null]], "Gen3 Query Class": [[7, null]], "Gen3 Submission Class": [[8, null]], "Gen3 Tools": [[9, null]], "Gen3 Workspace Storage": [[13, null]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Indexing Tools": [[11, null]], "Indices and tables": [[2, "indices-and-tables"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Metadata Tools": [[12, null]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, null]]}, "docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "indexentries": {"_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager", false]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods", false]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS", false]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create", false]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases", false]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record", false]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias", false]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases", false]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest", false]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get", false]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases", false]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record", false]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum", false]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page", false]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params", false]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest", false]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls", false]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd", false]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait", false]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update", false]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases", false]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record", false]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest", false]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider", false]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ", false]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create", false]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens", false]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children", false]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID", false]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url", false]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy", false]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create", false]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases", false]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank", false]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path", false]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job", false]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version", false]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list", false]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program", false]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project", false]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record", false]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time", false]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl", false]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR", false]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete", false]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias", false]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases", false]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids", false]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file", false]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations", false]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path", false]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node", false]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes", false]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object", false]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program", false]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project", false]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record", false]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record", false]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records", false]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download", false]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download", false]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download", false]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest", false]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single", false]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url", false]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable", false]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager", false]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus", false]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time", false]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint", false]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node", false]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record", false]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name", false]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name", false]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size", false]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size", false]], "gen3.tools": [[9, "module-gen3.tools", false]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download", false]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest", false]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest", false]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest", false]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest", false]], "gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth", false]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File", false]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index", false]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs", false]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata", false]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object", false]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query", false]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission", false]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage", false]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get", false]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get", false]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token", false]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts", false]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases", false]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records", false]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all", false]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node", false]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token", false]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema", false]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix", false]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths", false]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version", false]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output", false]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url", false]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs", false]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary", false]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest", false]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects", false]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record", false]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc", false]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records", false]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page", false]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats", false]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status", false]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls", false]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids", false]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version", false]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version", false]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version", false]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions", false]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params", false]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query", false]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID", false]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT", false]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT", false]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname", false]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest", false]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE", false]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy", false]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy", false]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy", false]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest", false]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object", false]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest", false]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs", false]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load", false]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest", false]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls", false]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path", false]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest", false]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS", false]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5", false]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum", false]], "module": [[9, "module-gen3.tools", false], [10, "module-gen3.tools.download.drs_download", false], [11, "module-gen3.tools.indexing.download_manifest", false], [11, "module-gen3.tools.indexing.index_manifest", false], [11, "module-gen3.tools.indexing.verify_manifest", false], [12, "module-gen3.tools.metadata.ingest_manifest", false]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id", false]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id", false]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type", false]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project", false]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint", false]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID", false]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query", false]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query", false]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query", false]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls", false]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download", false]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token", false]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects", false]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm", false]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path", false]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE", false]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time", false]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status", false]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file", false]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record", false]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl", false]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER", false]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update", false]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases", false]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank", false]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record", false]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time", false]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload", false]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file", false]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid", false]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url", false]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS", false]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access", false]]}, "objects": {"gen3": [[9, 3, 0, "-", "tools"]], "gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "terms": {"": [1, 4, 8, 10, 11], "0": [5, 7, 8, 11], "0084667": [], "013444": [], "0248635": [], "0791543": [], "0a80fada010c": 11, "0a80fada096c": 11, "0a80fada097c": 11, "0a80fada098c": 11, "0a80fada099c": 11, "1": [4, 5, 8, 10, 11], "10": [5, 7, 11], "100": [8, 11], "1125998": [], "11e9": 11, "120715": [], "1329331": [], "1468077": [], "165499": [], "1724170637": [], "1726237606": [], "1726237962": [], "1728059996": [], "1728059997": [], "1728061593": [], "1728061594": [], "1729113765": [], "1729629354": [], "1729629627": [], "1732401632": [], "1732401820": [], "1732401821": [], "1733254905": [], "1733263055": [], "1733431971": [], "1733932163": [], "1734031012": [], "1734031066": [], "1734031067": [], "1734474422": [], "1734474533": [], "1734474534": [], "1734644651": [11, 12], "2": [5, 11], "2023835": [], "2310405": [], "24": [11, 12], "244287": [], "2511687": [], "255e396f": 11, "256328": [], "284675": [], "3": [5, 11], "30": 8, "313695": [], "33": 5, "333": 5, "3410327": [], "343434344": 11, "363455714": 11, "3686683": [], "3697538": [], "3759608": [], "4": [5, 11], "4401503": [], "450c": 11, "4714": 8, "473d83400bc1bc9dc635e334fadd433c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "5": [0, 5], "50": 7, "5117643": [], "518752": [], "543434443": 11, "561932": 11, "5833347": [], "6129959": [], "6138082": [], "6146133": [], "653854": [], "6f90": 8, "7789228": 12, "7d3d8d2083b4": 11, "8366766": [], "8420": 8, "848236": [], "8493605": [], "8535511": [], "863009": [], "8712757": [], "9082778": [], "93d9af72": 11, "998105": [], "9a07": 11, "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "AND": 5, "Be": 1, "But": 5, "By": 11, "For": [1, 5, 6, 7, 8, 9, 11], "IF": 11, "If": [0, 1, 7, 11, 12], "In": 10, "It": 10, "NOT": 12, "OR": 5, "One": 7, "Such": 9, "THE": 11, "That": 3, "The": [0, 1, 2, 3, 5, 8, 10, 11], "There": 11, "These": 9, "To": 11, "Will": [4, 7, 10], "_get_acl_from_row": 11, "_get_authz_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_for_row": 12, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "_guid_typ": 12, "_manag": [2, 9, 10], "_query_for_associated_indexd_record_guid": 12, "_ssl": [3, 4, 5], "a5c6": 11, "ab167e49d25b488939b1ede42752458b": 3, "about": [2, 3], "abov": 11, "access": [0, 1, 3, 7, 10], "access_method": [2, 9, 10], "access_token": 0, "accesstoken": 0, "acl": [2, 3, 9, 11], "across": 11, "act": 0, "action": [9, 11], "actual": 11, "ad": 3, "add": [3, 5], "addit": [3, 5, 10, 11], "admin": [5, 11], "admin_endpoint_suffix": 5, "after": 10, "against": [3, 7, 8, 11, 12], "algorithm": 3, "alia": [3, 5], "alias": 5, "aliv": 7, "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "allow": [0, 6, 8, 10, 11, 12], "allowed_data_upload_bucket": 1, "along": 2, "alreadi": 9, "also": 1, "altern": [5, 11], "alwai": 5, "ammount": 12, "amount": [1, 9], "an": [0, 3, 6, 8, 10, 11], "ani": [0, 5, 10, 11], "anoth": 5, "api": [0, 5, 8, 11], "api_kei": 11, "appli": 7, "appropri": 13, "ar": [5, 7, 8, 9, 10, 11], "arbitrari": 0, "argument": [0, 13], "arrai": 8, "asc": 7, "assign": 9, "assist": 10, "associ": [3, 5], "assum": 11, "async": [3, 4, 5, 9, 11, 12], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_create_record": [2, 3], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_download_object_manifest": [2, 9, 11], "async_get": [2, 5], "async_get_alias": [2, 5], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_url": [2, 3], "async_query_urls_from_indexd": [2, 9, 12], "async_run_job_and_wait": [2, 4], "async_upd": [2, 5], "async_update_alias": [2, 5], "async_update_record": [2, 3], "async_verify_object_manifest": [2, 9, 11], "asynchron": [3, 4, 5], "asyncio": [11, 12], "asyncron": 5, "attach": [3, 5], "attempt": 11, "attribut": [10, 11], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "authbas": 0, "authent": 0, "author": 1, "authz": [0, 1, 2, 3, 9, 10, 11], "auto": [0, 2], "automat": 0, "avail": [1, 2, 10, 11], "az": 1, "b": [5, 11], "b0f1": 11, "bar": 10, "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "baseid": 3, "basic": [3, 11, 12], "batch_creat": [2, 5], "batch_siz": 8, "behalf": 0, "behavior": 11, "being": 3, "belong": 8, "below": 11, "blank": 3, "blob": [5, 7], "bodi": 3, "bool": [4, 5, 8, 10, 11, 12], "boolean": 3, "bownload": 10, "broad": 9, "broken": 9, "bucket": [1, 6], "bundl": 10, "byte": 10, "c": [5, 11], "cach": 10, "cache_hosts_wts_token": [2, 9, 10], "call": [10, 13], "can": [0, 3, 4, 8, 11, 12], "capabl": 9, "case": [0, 10], "categori": 9, "ccle": 8, "ccle_one_record": 8, "ccle_sample_nod": 8, "cdi": 7, "chang": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "child": 10, "children": [2, 9, 10], "chunk_siz": 8, "class": [0, 2, 10, 11, 13], "cli": 10, "client": [0, 3], "client_credenti": 0, "client_id": 0, "client_scop": 0, "client_secret": 0, "code": [2, 8], "column": [11, 12], "column_to_use_as_guid": [2, 9, 12], "columna": 11, "columnb": 11, "columnc": 11, "com": 7, "comma": 11, "command": [10, 11], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "commons_url": [2, 9, 10, 11, 12], "complet": [4, 11], "complex": 7, "concat": 11, "concurr": [11, 12], "configur": 1, "connect": 12, "consist": 3, "constructor": 0, "contain": [0, 2, 5, 8, 9, 10, 11, 12], "content": [3, 13], "content_created_d": 3, "content_updated_d": 3, "continu": 10, "control": 3, "conveni": 10, "copi": [2, 13], "coroutin": 11, "correspond": 3, "count": 3, "crdc": 0, "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_blank": [2, 3], "create_index_key_path": [2, 5], "create_job": [2, 4], "create_new_vers": [2, 3], "create_object_list": [2, 9, 10], "create_program": [2, 8], "create_project": [2, 8], "create_record": [2, 3], "created_tim": [2, 9, 10], "creation": [3, 11], "cred": 3, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "csv": [8, 11, 12], "curl": [0, 2], "current": [6, 8, 10], "current_dir": [2, 9, 11], "custom": 11, "d": 5, "d70b41b9": 8, "data": [0, 1, 3, 5, 7, 8, 10, 11], "data_spreadsheet": 8, "data_typ": 7, "data_upload_bucket": 1, "dataa": 11, "datab": 11, "databas": 5, "datacommon": 0, "datafil": 10, "datamanag": 10, "date": 3, "datetim": [1, 3, 10], "dbgap": 12, "dcf": 8, "def": 11, "default": [0, 1, 3, 7, 8, 11, 12], "defin": [5, 8, 10], "delai": 4, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_all_guid": [2, 9, 11], "delete_fil": [1, 2], "delete_file_loc": [1, 2, 6], "delete_index_key_path": [2, 5], "delete_nod": [2, 8], "delete_object": [2, 6], "delete_program": [2, 8], "delete_project": [2, 8], "delete_record": [2, 3, 8], "delete_unpacked_packag": 10, "delimet": [11, 12], "delimit": 11, "demograph": 8, "deprec": 1, "descript": [3, 5], "desir": 11, "dest_path": 13, "dest_urlstr": 13, "dest_w": 13, "dest_wskei": 13, "detail": [2, 7, 10], "determin": [10, 11, 12], "dev": 11, "dict": [3, 4, 5, 10, 11, 12], "dictionari": [3, 4, 5, 7, 8], "did": 3, "differ": 5, "direct": 0, "directori": [10, 11], "disabl": 10, "discoveri": 10, "disk": 13, "dispatch": 4, "dist_resolut": 3, "distribut": 3, "doc": [7, 10], "docstr": 2, "document": [1, 3], "doe": [0, 12], "domain": [11, 12], "done": 4, "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "download_files_in_drs_manifest": [2, 9, 10], "download_list": 10, "download_manifest": 11, "download_singl": [1, 2], "download_url": [2, 13], "downloadmanag": [2, 9, 10], "downloadstatu": [2, 9, 10], "dr": [2, 9], "drs_download": 10, "drs_hostnam": 10, "drsdownload": 10, "drsobjecttyp": 10, "e": [5, 10], "e043ab8b77b9": 8, "each": [3, 8, 10, 11], "effici": 9, "eg": 3, "either": 8, "elasticsearch": 7, "els": [0, 12], "elsewher": 12, "empti": [8, 11], "enabl": 11, "end": [5, 10], "end_tim": [2, 9, 10], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 10, 13], "entir": 8, "entri": [3, 11], "env": 0, "environ": 0, "equal": 7, "error": [10, 11, 12], "error_nam": 11, "etc": 8, "even": 11, "everi": [9, 11], "everyth": 11, "ex": [0, 11, 12], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "exclud": 3, "execut": [7, 8, 11], "exist": [1, 3, 5, 6, 9, 12], "expect": [5, 9, 11], "experi": 8, "expir": [0, 1], "expires_in": 1, "export": [8, 10], "export_nod": [2, 8], "export_record": [2, 8], "f1f8": 11, "factori": 10, "fail": [8, 10], "fals": [3, 5, 6, 10, 11], "featur": [1, 6], "fenc": [0, 1], "fetch": 0, "field": [3, 5, 7, 11, 12], "fieldnam": 11, "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "file_nam": [1, 2, 3, 9, 10, 11], "file_s": [2, 9, 10, 11], "file_st": 3, "fileformat": 8, "filenam": [0, 8, 10, 11, 12], "files": 10, "fill": 12, "filter": [5, 7], "filter_object": 7, "first": [7, 8], "flag": 11, "folder": 11, "follow": [0, 11], "forc": 11, "force_metadata_columns_even_if_empti": 11, "form": 13, "format": [3, 5, 8, 11], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "func_to_parse_row": [11, 12], "function": [2, 3, 4, 5, 9, 10, 11, 12], "g": 10, "gen3": [10, 11, 12], "gen3_api_kei": 0, "gen3_oidc_client_creds_secret": 0, "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "gen3fil": [1, 2], "gen3index": [2, 3], "gen3job": [2, 4, 10], "gen3metadata": [2, 5], "gen3object": [2, 6], "gen3queri": [2, 7], "gen3submiss": [2, 8], "gen3wsstorag": [2, 13], "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "get_access_token": [0, 2], "get_access_token_from_wt": [0, 2], "get_alias": [2, 5], "get_all_record": [2, 3], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_fresh_token": [2, 9, 10], "get_graphql_schema": [2, 8], "get_guid_from_fil": 12, "get_guids_prefix": [2, 3], "get_index_key_path": [2, 5], "get_latest_vers": [2, 3], "get_output": [2, 4], "get_presigned_url": [1, 2], "get_program": [2, 8], "get_project": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_statu": [2, 4], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "giangb": 11, "github": [2, 7], "give": 1, "given": [0, 3, 4, 5, 8, 10, 12, 13], "global": [4, 5], "good": 3, "grant": 0, "graph": 8, "graphql": [7, 8], "graphql_queri": [2, 7], "group": 3, "guid": [1, 2, 3, 5, 6, 9, 11, 12], "guid_exampl": 11, "guid_for_row": 12, "guid_from_fil": 12, "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "guppi": 7, "ha": [0, 11], "handl": [3, 10], "hardcod": 0, "has_vers": 3, "hash": [3, 11], "hash_typ": 3, "have": [5, 11], "header": 11, "healthi": [3, 4, 5], "help": 11, "helper": 2, "hit": 11, "host": 10, "hostnam": [2, 9, 10], "how": [8, 11], "howto": 10, "http": [0, 7, 11, 12], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "id": [0, 1, 3, 5, 10, 11], "idea": 3, "identifi": [3, 5, 9, 11], "idp": 0, "illustr": 11, "immut": 3, "implement": 0, "implic": 11, "import": 11, "includ": [0, 3], "indent": 10, "index": [0, 2, 5, 9], "index_manifest": 11, "index_object_manifest": [2, 9, 11], "indexd": [1, 3, 6, 10, 11, 12], "indexd_field": [11, 12], "indexd_record_page_s": [2, 9, 11], "indexed_file_object_guid": 12, "indic": [0, 11], "infil": 10, "info": [3, 11], "inform": [2, 3, 10], "ingest": [2, 9], "ingest_manifest": 12, "initi": [0, 10], "input": [4, 10, 11], "input_manifest": 11, "instal": [0, 2, 11], "instanc": [1, 3, 6, 7, 8, 9, 10], "instead": [1, 7, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "integ": [1, 3, 8], "intend": 0, "interact": [1, 3, 4, 5, 6, 8, 13], "interest": 10, "interpret": 0, "introspect": 8, "involv": 9, "is_healthi": [2, 3, 4, 5], "is_indexed_file_object": 12, "isn": 1, "issu": 0, "its": [1, 3], "job": 2, "job_id": 4, "job_input": 4, "job_nam": 4, "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "just": [5, 11, 12], "jwt": 0, "kei": [0, 3, 5, 13], "know": 11, "known": 10, "kwarg": [3, 4, 5], "l": [2, 13], "larg": 9, "last": 10, "latest": 3, "least": 3, "level": 6, "librari": 11, "like": [3, 5, 9, 11, 12], "limit": [1, 3, 5, 12], "linear": 4, "linux": 10, "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "list_job": [2, 4], "live": [11, 12], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "local": [0, 13], "locat": [1, 6], "lock": 12, "log": [8, 10, 11, 12], "logic": [5, 12], "loop": 11, "ls_path": [2, 13], "maco": 11, "made": 3, "mai": [0, 9, 11], "main": 10, "make": [9, 11], "manag": [1, 5, 10], "mani": [8, 11], "manifest": [2, 8, 9, 10, 11, 12], "manifest_1": 10, "manifest_fil": [11, 12], "manifest_file_delimit": [11, 12], "manifest_row_pars": [11, 12], "map": [0, 11], "mark": 8, "master": 7, "match": [3, 5, 12], "max": 5, "max_concurrent_request": [2, 9, 11, 12], "max_presigned_url_ttl": 1, "max_tri": 8, "maximum": [11, 12], "md": [5, 7, 10, 12], "md5": [2, 3, 9, 11], "md5_hash": 11, "md5sum": [2, 9, 10], "mean": 8, "mechan": 3, "merg": 5, "metadata": [2, 3, 6, 9, 11], "metadata_list": 5, "metadata_sourc": 12, "metadata_typ": 12, "metdata": 12, "method": [1, 7, 10], "minimum": 10, "minut": 0, "mode": 7, "modul": [2, 10, 11], "more": [2, 5, 7, 9, 10], "most": 9, "mostli": 2, "multipl": [8, 11], "must": [1, 5], "my_common": 10, "my_credenti": 10, "my_field": 7, "my_index": 7, "my_program": 7, "my_project": 7, "name": [3, 4, 8, 10, 11, 12, 13], "namespac": [0, 12], "necessari": [3, 5], "need": [3, 7, 10, 11], "nest": 5, "net": 11, "never": 0, "new": [0, 3], "node": 8, "node_nam": 8, "node_typ": 8, "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "note": [0, 3, 11, 12], "noth": [3, 6], "now": [1, 8], "num": 5, "num_process": 11, "num_total_fil": 11, "number": [3, 7, 8, 11, 12], "o": 0, "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "object_id": [1, 2, 9, 10], "object_list": 10, "object_typ": [2, 9, 10], "objectid": 10, "obtain": [0, 10], "occur": 10, "off": 5, "offset": [5, 7], "oidc": 0, "old": 3, "one": [3, 5, 10, 11], "onli": [3, 5, 7, 8, 10, 11], "open": [8, 10, 11], "open_project": [2, 8], "openid": 0, "opt": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "order": [0, 8], "ordered_node_list": 8, "org": 10, "otherwis": 10, "output": [4, 5, 11, 12], "output_dir": 10, "output_filenam": [11, 12], "overrid": [0, 11, 12], "overwrit": 5, "own": 0, "packag": 10, "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "pagin": 3, "parallel": 11, "param": [3, 5, 8, 10], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "pars": [10, 11, 12, 13], "parser": [11, 12], "particular": 0, "pass": [0, 7, 8, 10], "password": [11, 12], "path": [0, 1, 5, 10, 11, 13], "path_to_manifest": 11, "pattern": [3, 12], "pdcdatastor": 11, "pend": 10, "per": [11, 12], "peregrin": 8, "permiss": 10, "persist": 9, "phs0001": 11, "phs0002": 11, "pick": 1, "pla": 11, "place": 11, "planx": 11, "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "popul": [10, 12], "posit": [1, 7], "possibl": 10, "post": [0, 11], "pprint": [2, 9, 10], "prefix": 3, "presign": 1, "pretti": 10, "prev_guid": [2, 9, 11], "previou": [3, 11], "previous": 4, "print": [8, 10], "process": 11, "processed_fil": 11, "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "program": [8, 11], "progress": [8, 10], "project": [8, 11], "project_id": [7, 8], "protocol": 1, "provid": [0, 1, 3, 5, 7, 8, 12], "public": [3, 5], "put": 0, "py": 11, "python": [2, 9, 11], "python3": 11, "python_subprocess_command": 11, "queri": [1, 2, 3, 5, 8, 11, 12], "query_str": 7, "query_txt": [7, 8], "query_url": [2, 3], "quickstart": 2, "rather": 0, "raw": [7, 11], "raw_data_download": [2, 7], "rbac": 3, "read": [3, 5, 11], "readm": 2, "reason": 10, "record": [1, 3, 5, 7, 8, 11, 12], "refresh": [0, 10], "refresh_access_token": [0, 2], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "regist": 8, "regular": 7, "relat": 9, "remov": [1, 6, 11, 13], "replac": 11, "replace_url": 11, "repo": 2, "repres": [3, 5, 10], "represent": [1, 3], "request": [0, 1, 3, 5, 8, 11, 12], "requir": 10, "resolv": 10, "resolve_object": [2, 9, 10], "respect": 7, "respons": [0, 1, 3, 4, 5], "result": [1, 8, 10, 11], "retri": 8, "retriev": [1, 8, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "return_full_metadata": 5, "rev": 3, "revers": 8, "revis": 3, "right": 1, "rm": [2, 13], "rm_path": [2, 13], "root": [11, 12], "row": [7, 8, 11, 12], "row_offset": 8, "rtype": 3, "run": [8, 11], "s3": [1, 10, 11], "safe": 11, "same": [5, 11, 13], "sampl": [8, 10], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "save": 10, "save_directori": 10, "schema": 8, "scope": [0, 1], "screen": 8, "script": 2, "search": [0, 2, 3], "second": [1, 4], "secret": 0, "see": [7, 10, 11], "self": 10, "semaphon": 12, "semaphor": 12, "separ": [0, 11], "server": 10, "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "service_loc": [3, 4, 5], "session": 11, "set": [0, 1, 5, 10], "setup": 2, "sheepdog": 8, "should": [0, 8, 11], "show": 10, "show_progress": 10, "shown": 11, "sign": 1, "signpost": 3, "similar": 10, "simpl": 3, "simpli": 11, "sinc": 3, "singl": [1, 5, 8], "size": [2, 3, 9, 10, 11], "skip": 8, "sleep": 4, "so": 11, "some": [0, 2], "someth": 11, "sort": 7, "sort_field": 7, "sort_object": 7, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "space": [0, 11], "specif": [5, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "spreadsheet": 8, "src_path": 13, "src_urlstr": 13, "src_w": 13, "src_wskei": 13, "ssl": [3, 4, 5], "start": [4, 7, 8, 10], "start_tim": [2, 9, 10], "static": 10, "statu": [2, 4, 9, 10], "storag": [1, 2, 6], "store": [1, 3, 10], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "string": [0, 3, 5, 11, 13], "strip": 11, "sub": 8, "subject": [7, 8], "submiss": 2, "submit": [8, 11], "submit_additional_metadata_column": 11, "submit_fil": [2, 8], "submit_record": [2, 8], "submitter_id": 7, "success": 10, "successfulli": 10, "suffici": 3, "suppli": 3, "support": [0, 1, 5, 8, 11], "sure": 1, "synchron": 11, "syntax": 7, "system": [6, 7, 8, 9], "t": [1, 5, 11], "tab": 11, "task": 9, "temporari": 11, "test": 11, "test1": 11, "test2": 11, "test3": 11, "test4": 11, "test5": 11, "text": [1, 7, 8], "than": [0, 5], "thei": [0, 10], "them": [10, 11], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "those": 11, "thread": 11, "thread_num": 11, "threadcontrol": [2, 9, 11], "through": [8, 11], "tier": 7, "time": [1, 3, 8, 10, 11], "timestamp": 10, "tmp_folder": [2, 9, 11], "token": [0, 10], "tool": 2, "total": 11, "treat": [1, 5], "tree": 10, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "try": 0, "tsv": [8, 11, 12], "tupl": [0, 3, 11, 12], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "typic": 10, "uc": 7, "unaccess": 7, "under": [0, 8, 13], "uniqu": [1, 5], "unknown": 10, "unpack": 10, "unpack_packag": 10, "until": [4, 10], "up": [5, 9], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "update_blank": [2, 3], "update_record": [2, 3], "updated_tim": [2, 9, 10], "upload": [1, 2, 3, 8, 13], "upload_fil": [1, 2], "upload_file_to_guid": [1, 2], "upload_url": [2, 13], "url": [1, 2, 3, 9, 10, 11, 12, 13], "urls_metadata": 3, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "usag": 11, "use_agg_md": 5, "user": [0, 10, 12], "user_access": [2, 9, 10], "usual": 12, "utcnow": 1, "util": 9, "uuid": [1, 3, 8], "uuid1": 8, "uuid2": 8, "valid": [3, 7], "valu": [0, 1, 3, 5, 7, 10, 11], "value_from_indexd": 11, "value_from_manifest": 11, "variabl": [0, 7, 8], "variou": 2, "verbos": [7, 8], "verif": 11, "verifi": [2, 9], "verify_manifest": 11, "verify_object_manifest": 11, "version": [3, 4, 5], "vital_statu": 7, "w": 13, "wa": 0, "wai": 10, "wait": 4, "want": [0, 3, 8], "warn": 11, "we": [3, 11], "web": 0, "what": 5, "when": [0, 3, 7, 10, 12], "where": [0, 3, 5, 11, 12], "whether": [3, 4, 5, 8, 11, 12], "which": [8, 10], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "whose": 5, "within": [0, 2, 9], "without": [3, 5], "won": 5, "work": [0, 10], "workaround": 11, "worksheet": 8, "workspac": [0, 2], "wrapper": 10, "write": 11, "ws_urlstr": 13, "wskei": 13, "wss": 13, "wt": [0, 10], "x": 11, "xlsx": 8, "you": [0, 3, 8, 11], "your": 0}, "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "titleterms": {"": 2, "auth": 0, "class": [1, 3, 4, 5, 6, 7, 8], "document": 2, "download": [10, 11], "dr": 10, "file": 1, "gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "helper": 0, "index": [3, 11], "indic": 2, "ingest": 12, "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "sdk": 2, "storag": 13, "submiss": 8, "tabl": 2, "tool": [9, 10, 11, 12], "verifi": 11, "welcom": 2, "workspac": 13}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index b5d60795f..d67d1b2df 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -381,7 +381,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1734474533.863009.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1734644651.561932.log')[source]

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index 3f47dd6f8..be7bef327 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1734474534.1125998.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1734644651.7789228.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters:
From 3f8c02d081c9bcca90de01a0e35243544b147133 Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Thu, 19 Dec 2024 15:47:03 -0600 Subject: [PATCH 7/8] One last change --- gen3/tools/indexing/post_indexing_validation.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gen3/tools/indexing/post_indexing_validation.py b/gen3/tools/indexing/post_indexing_validation.py index 5d3c50ad4..4f7626560 100644 --- a/gen3/tools/indexing/post_indexing_validation.py +++ b/gen3/tools/indexing/post_indexing_validation.py @@ -122,7 +122,6 @@ def read_records_from_manifest(self, manifest): url_pattern = r"^[a-zA-Z][a-zA-Z0-9+.-]*://[^\s/]+(?:/[^\s]*)*$" for row in rows: - url_parsed = False size = row["size"] guid = row[guid_col] for acl in row["acl"].split(" "): @@ -162,10 +161,6 @@ def read_records_from_manifest(self, manifest): size, ) self.record_dict[key] = record - url_parsed = True - - if url_parsed == False: - logger.warning(f"No url parsed for record {guid}") def check_records(self): """ From d987c7f92ddc1f5a574a1fb94b394edc60907c60 Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Thu, 19 Dec 2024 21:48:07 +0000 Subject: [PATCH 8/8] Apply automatic documentation changes --- docs/_build/doctrees/environment.pickle | Bin 447288 -> 447288 bytes docs/_build/doctrees/tools/indexing.doctree | Bin 100192 -> 100197 bytes docs/_build/doctrees/tools/metadata.doctree | Bin 35909 -> 35909 bytes docs/_build/html/searchindex.js | 2 +- docs/_build/html/tools/indexing.html | 2 +- docs/_build/html/tools/metadata.html | 2 +- 6 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index e3b8c17925a3e42249f50857ae4742d757407d7f..e11c610d342c7a9e478a69d52e1c75fda4550320 100644 GIT binary patch delta 9284 zcma(%d0dspwa(m)eHXm!n~Jhs1QA6L+(1yoEgD4@1<@MWKjl<6(Y z%oWS)TCGm2s>>^WwyL_a)#@Q+iO*Ejm6ey(mEjn(Wf|&f%j)Y^;!vUKP%B?kTT)YB zRfQ((BfK3#yHREy9lgGn4jE!~;`1l0sjX`@-{-WmP?3yBoZ)64BYE9o5O+Y*KMEcD zOp@2nc`kA#lJk&roJABQqmvFv8DjMqjD%$06MAC3lJS5u3>Y>^cCqYOBk5yHVvRuq zT;%g)Y@}8XYmN;P%QF+rkRf7WX38v*j$M_1&I*xI_&3cY;f6WGtp`%00 zP8{RDnVmVtoie*{tczo=9P8tl8^^d`W_OJi|MuA8Gdyp+in(He_?r4+YqcZ`5;?6c1wp0cSsz*@$Q$qyB7uRGM;~)AG8*a)Y zy{yI*Nba-Eroc4rXLDre9VaLij@x2md^{bNir z+cn0Y?MNJUl^!UMiSw9ZPBv(ypAq>BwC@5SZ4lX4Z8*ES)N?HIe1 zJQ7$~Y8~4#A&MPIEoLk=3MG1$F>WF28ym%xab>s`uAiP(!dk~gu@h;j*HTnzJ_TVHOCG56XoOpGF|`F6C;FY^VCc-wYNvPool z_74JKcVBa|t(wv+2)H?#IN4sBHXude$DG*iN$$3`94`u4CSqY>c@^NbfOxaE!@jmZ zD3e3D?ZfY$MY2lEkf+-7l z@(SLA#S~%&ne}LqXOUu&U6ELkQ;|-Q$u8URB3H@^r}(hH72k&H^%&73siAC6X)e_4 zP@{UH)IvDrW6NCjp~OIhFsQs+6-2Lap;9GlrLV-`U^7O!%1kBsRzsCUSoDez+kt9~ zCO0Vzv7mB)+o_rxsVi-sodN4=($saB3PG1 zJ=k~8dE4mPKdGiZegPB7=fSiF!9Wsqv)y^=Q&srcx_*fPpNDnSd9mwN9L%In@_caDbuJr{#u|7wjM+^aM$w)n>Ls#RWfY~)ry_$ry|ZE>5=sV1C% z*(EXIR8b0bgR?t{w{2SU36u@e2@-M#UEFy> z5Vq}a^+;s*uG><)d)H7X9Y?jcle<5Cg5wIQS-6ZNDYe+B`8Di%)8(Fly-_0ulSZ@qN|s%iB@s zjSsO0;l!dmY=1o4?=Wcl@*~xlI;TH0cM^BoqfgFDo&9SJwwvcWC9(Z21B?6I|L~>n zB%OIJDiTlf{#LsVur&-+8eeWJISKE-f4CJp|Jbcmsix8gpRE@xirlPaRX7Tz%0 zs;*xZ99!WHDs<%LTm97|Ib-3C2soHYo!Gc7-nPTHPD$lgzrH7tU;Oiq)Z=cqeDkE+ zwu9d~Q+9BM1rMn(mUBCoeLowMa{E<+h!3=!#z;;4_j`g8LG-vANSSiTf`?fs`%_o$ z>qli}MclKH*N^(J%w-1Kp8JZxdFx&K@LwFaJ=@bMc)Y*9*c(j2?G$#SOgz#?_2p1(OKmxw8-Ucq^|O9ybyLEyuM^3Yy3LWS=+&PPq^R zfS;3hfJ+67#mteOPBWJye=HdfMC^WZgT@^on z@|9{sH;i~brlg<}k+gzdyQye#DDhLZT6Urno5SrhD{YaES@<$xJEiW*KcdM6IRVe1 z^3e!_t?$VYRcH;+c1){NdSe`M;tK>^VBoe2t-InGi|b)AT^9SjWeID!FOm=a@#M&O zapd?Jh71vBA-_QIvy5LKIOI1gJk!kKQbArY#NDDtxt&1f$AK~$pOr}EfvE?jCS?n>x*;_(;_ zNRNP$jXE#(`)S_F>*?eJX#+N|B~cFRPH=D{7R{~;D}l5%Xf#q0_w?9n!T07Pm|+Ps-R=K$9z=IE+u$?iA5hMv5Q#NN`rEtj65eb zd0SJ8%K7#ZGhnH@D_JWDo{tV1kPcI7R`NajBomOS1CuY+DVxpYtP~HH!44k`QgfBV zecTcT<%MdyD>9ox=|l~=rW(gfi#zPgBmTf8;5848Ut2>isbUOtltZEgw@X@QcrP5w zOIJ^3s>xdZJn>gijF>@PDORU~)%lZZX#XDjJYMrLZGWCm{gvLANgZLccUbuCzMQq} z6DIcY2134ryY!BGzIj?LZ7O%iI{2Utr$9M>oHyd%o3;w~)Btc?#e@XVs z@wiMp$N6|X!2T&Pe##A3K4An;2W|x0FeuF=RO#5NvZTflAb1R;h9vjBK~76ccpv&+ zV%Qb%WKlf<^u!Zt7E@2S_B8cS{d8UEf6x$f3BKK-r8#!y~B| zTwhGxl#9D@b4joF5MMDs?%O#_?#ekhk!l{KFgwE$0?EvrLJo0a_u0sgp)oB9-_F71iJJCf>&07 z7M&sZL=ounKa)EWg`x|@sJ#3!xuS~j67ZdUzy&+XZ(TWbb;S$m1M31MZ?6@srE zjxs#yq4FGd!1I^&uFCpPi5D&6kCS})c_H6PychGSc%kHni3du)vv{B62aD%PzDB%G z@^#{Ik{=-6Ciz6P?fl`Qt0R*)bu`um-~WY7Dc~<6?EQ~DQvN@06dY6`tn;Db*(py2 zc1-D)8-Q*E2s-R(?*FQ_`!PzlK)~a$kAtI%%O! zGIr)}CH+g%2&VEadQPQt$xJ~k0$wN5bP2MG)%7z|NP9R_;Znb+q61@v~0KFQT zLHd+5Eq&xm29%Ux^q33jQSL<27%k~ihK;1VHM4p48%6vYMJyXd>KjGo8%5$9Mcx}l z+8agI8^vN8MN%5YavMd)8!_P?v9yTv!s=L>ir{y#bc%sbVeT9WSyO2MIA13*9_G=b zhYYdT!$>GiM?qm69gQPijH6@7eb^s|zTb!2aWq#iK4uX3X)Nx+x$$(kWN5*Zf{d;4 zbR;UCh^Ltd`r_$y1XB`d3h9QQCD0rM9SL+A0(~N#k6=k6EkN*2BF#haAW_KDO*oGz z%8_tlER7NG&7&sjE}5gDBpv5^`zzvw5??3P|8DR|qLb0e90^*JXc|%%lIT=9ZanlR zqA`5Mu{-PBp>PZ}NWsL%407o}gXf^pwlQ=Pnz=cKPC*czOta*mT>SJQtvTMEt_B&& zZrGoU0qBMwk}+=G5Sv1$BB)8BSqKiL&}?a9F%McCWc@F31JhV)Kw;WgG2721I69VQ zBeQp`SXV|WCgVq_Nu{|6-bo{DPkUZy*_(;ZDn3;{C zYZ*uJEs{cY>Yy_qWbgDs! z-;9w}IpQvfaev(8Z+U_e#B^VNq<3~q2QKNt#Nu?CBVF9h3uYLcN~h71@eX)o(1h17 zIWpQ02@`1{&bxjh2DcwhPsFn92j>hr1HrruS||s$%TR60q{b)LcM9+*ABOxlW9JJqm$_@sr!iQPV^N&RH(rgHU>l%lHjr%ad(9J9_*Y#XW{reQ)r$X zZ-AXSIG2|%UZU1brB0IT0!hl0KZ(^M18Oz7nyzVyIi^gf%6Rdv|b!a z8SwE8Iu&ldqVrY*&<7vRprxoZJdYL`gwpg;;KiHVjy&8@eQ-68E|8;V!p>P(;Fbb9 zT09zHO+NLNqS;c^mXDiP7?~>>efgNHK1iNPpOXAVlK=aebh2bDk&O7+G?Bj(#+?=tRkQ#lgT=58m_#2c4cn=SiJkNS(yF zSdo2DJ(rdsxHuQPdmn_%qYDw#&!Z($8DL8>^@D%Uv#;aVk}+#O9WOPuOGev#Y~CW= zZIUrCpH9$=o5yZB;kygyNZ4+q!|V-12;XrCr%Ua9l5u?we&=fsqR!il)CW!mQdj$0 z24MD6*jESOf1aYHa*Rz*>9dfIlZ^MI5$u1_@Qa1iO>)~EToVqzxsc9~{P!LFXjr+( zu6;&wn-^gV9Dt5RSoZ_q_B6J!0VsMJ<01S%0*iBLAOweFW5EmiP!!&I8uKHBpGm)# zEXIZ~0NWSiwnFh@T8yAz2{wiS*t&!kN#(DlvTG6c@*XHCqAT_M4rKm8`f;TQtqR6n z$;e!a5f=>H@_aLFUrOU}-`!izt=VAe#VWQSQ3g5ZQ%6NTWGSrbaSx2@KM z266LN_ZX!=P=o#t$yz z_jBn_N2SInM}|ldepPps3}3jCOovL`2fR|KA39V}r3pu1snP@^uvTe8q@9`-cC4*j zGb~;BTNG%Pr;cAGk3hCAD>Bb@e!j5D5nyqUS2sh~MzdKSBqN zMkCi-G)BDIKWNb`YQ{skAY2=cz!I*F5r?-mj3piOg_cmQ2eMB&*#4ZoEkdi4TA@N~ z{{-qTZ46_0bv}jI8i8qp+DCsg1;WiX*k7jQoaS_2TQFrT(*ltcreUi`0gpR!5|EBm$)T`3*&# zAb@rJBOnAF+Cz-g{7}J?z8I?K zk2h-oTmNtvHFrX1zSdvMSy zgP`*f2{wqgjx_|fIFS(CuzakfjE*s2)t2%Kb!0H$ z_ncr19d@~P?J#j)aV^9Dw~&S-flR{|AcQe?H*S?-yiAJLK<`mF6h^|JxkT$?_azEY z9%ZM3I2Gv39pH`z!LKN_ufY$<2;wK^##Q>^S6lH-S1;!YhR$-W@J#vPg!Y@W;`pn52Q5PA@rB;urmglRhEM?{M)bbZx zH#@g%8UD`-4mi0{7mUaA^^LkfZ=vBdtE_s}iVFVUBz>UYq>JW8PWxAM&Ax|#Ds zIos-0UQtznn;sKUQdz&M9K$F0Lt*`58i23nJF$9}E)KR@D;(NPhJq zhml3mM~OqK-U@S$;OD1yMZY3Y%TE<3yrTEVGbH1xK19zM)<7sLp}|sEb4Bkbv0q)$ z8zpwg!MlXKa4|--j)k@J>MU!@Dk~D|Yh)zE1nrmg!Tubpw>nkg=1buJCREp2U%xEj F{{S%xvJU_N delta 9157 zcma(%X+V|7wa(m)eHXdxn~EZE5fl`Gi%URI#4Q>{0R@*Ua*2ScSp<`43f>mXpy#F5 zinWO~Hf~?)=PqWkN!*jPskJ8BzP`L9rphLYXx^EbdoTLlpZABe&Y3gkoH;Y|U5>sJ za`c^$gMS-Q@3Ok0wyeCYwyf3WR#UfR^~#!>mDZ|Oo92a@npT@vt<_ptlT^N@rlh*A zva;2td%mt>&AL{bcG=3x3a5N!Re8nFR#q)XxfsxDa~WXitxGHwOKWjVSzYZ4=NLJ! zFP~T^lu^NAT~fm=BV+3y(m{i4E_{x}H8r)ZmV2Cb6>5_4h%-DKlStkJF^Joc^kbpp zSfAu|aGsm=NOJl($5kv#GJ5Es)Im0{fnZ2>r_dA2lZ^YEVZbm+vWw-!8%Z}?7;g;e zN%tATFS7gy}jIYC@X zJ#8{fzmp|o_j3fP7G(>PJKAKJb9IuSxU)@$tAENBRKK8J+?yvTzPKhs@6V=Iv7x4X z@{m=Vg2_F$8UH?D?hGL}65~Yi+S~up6du(#QHup`?cy zlS4=+8ZoaD=yBWlXqJ&)!gh^|VW-j;vmF^RENk3S7LKstM;)wB zApY#{2Zph}>0XNI*;`V1#tj!`@qhh8>UYaU1u z&#Tg2+a)Mw+A)M(sWm{ta*csym-{LY*Pd}oV0|+6V&AXzRp`1uN;99nhqSmf3a zc$`cWru^Y3` z83@fC#8Y|r8BaIr>|bM0ZeGA{!HFHa4Q$j`frl@DFX^mnVQ*eGD3Mp~9-=b4c_9!Q zx`|#feZgxH-4sP0`e*47x3PQFSJLIIYyDD%2bImcVN@!w^NPUL6y4xk3P+0#w(6UQ za>k+?QE)Jey0Ek@zRKZSe0!ksYyafsjL6Ubbw}!Pw_Cn_+HK|FcditVoP#QQnx;;B~W;t z7-5e$F)A59p*<9e1V=%{39TC(>?1m8awWd%6hc1!Ut(`{nucKA@-eKT(f}uYN$|FQ z;;t^zkw4p&&7mX;d7kXUy}s&eh9`~0LCXoPi;Cv|GGI9l&bSc+6ljQ_s<@Mn>=Is2 zYKtd%ENK;MP(z(Ft2pz+CpEA56Dq|p{WUa~5KO|M`J~oe4G1LP*lp;7i1!mpDk>34 ztLU|d9bFtw0_<8XyU>Zvk&cuGfgBkpPL=?}ph4ouau}3s)cLUAP4QLV z%pf0%@vwOviE$csfrAsUT;9$k)5TD14F=WSOfE^?`h_GOlFUS}hE62docXN!$;4gG zAM*^^C@16%3HIcZIQ6?c^0w4#%O|c<@5D6XBK6Ro2~5+etLilalO%jcJrk68QXOO$ zVXeQo0&6{eCOP@E)J-+dCRZd6J&4~n19i=XnC_Cf|A)Wc3;FBk5o{?uI*t{4LVqFk zW~t4->fz_e2_aO`CEb8?>bXVaD=9coViz#;3WIvFjI0%EypO5H<+zQ6ZLCO7)x3=0 zNyw+-Bp?}~RI!+H=%P{o_vZsZ_(9YNccE>?Sc1yZ$ygAcx|H0d+GDjf|!~aD4&wP%rJ_`$u@a zm*91gJ0W)H&3(i~!E7Q{{)8z9pSiN3sD=Z?2fNK&Z2f3CK+QaeyGYWw$vt7w5j-PI z#Tb0kVcds8q6WWDVyQ%nju1Sr#CVZ0p?`fY4T9TyaXXzqiq$4HcOQ3Xs_0TCWzcWM zCsZu#JVWxJ)ZOlodpoHiob*%ia6Dm07kx;6w4=ozlO8*|^iSlDM4|X1F{-b8O0L>P zc)Wk_7;w>n@|%~+YW)|)hZgfEMS*%yEih>C7tSE35uZn_D@u(=!h&M%nPCO|JhKLtM0TEreV5n%)$mBg5jdj5f zejM;jzzk6Q zB{M+rC(QuG-!=mjf8-2M{7f34x@JB_a__j1E47Fi3{X52PO8r4xxCW4qy(R_wQDNy z#pRG#9;qw7B16wNkW!k!g)kd4u?}?Xq^ z_`|zt_#*R5#wj1b%w#$dL2EKiM{qHjPL^ZFL0=LY!iO2Vv)&VmMpJ_nOnAZ=mp*Rr zmIi+}nr5Jho1^I@1hFa9EC=LaUz3)ck4JZdjNk*!7);cAAlCch^qsL?}Aq*O?>mRw~NCdBu=13IB)#~3~mRUoq!eD z0j`;JDuUUWv`7v-DMPg_iyEI^-xIka=IDF4t}SHhHUDB3XR#qi7&Eor}lxH8LO@j3eC84df+uPy4wRE zn=v;%+j4MI^+0Y8R$3=)%AwQr;sekf9^ldZG38?~$4-cvh})_YDkstc1V<;*=~DM0 z*PY-mzV+Kmib>Q2}-iB8AycP7z%Io<%fa&ayne>}3+Po^%C>jueF z=veXifs2!=I}W)vna)BmY6_i+VEq)~$7fUMT%;nOp|cUxJ|mdz5_sg|mg$6Z z=hA6NeV2>P=spDG;Xdkyxp~O!hTrDVWe8lS(&zNzJj#Smr_#xA^HrU%JpkSC(NtQB zN<;H$u|X(3GXi{gf7_Xl8>$;V&!=aN;C>&kvB? zqoGFCfnXQREHekB>x=g@IdV~1q4&B5L+!rdkrkLS?wdU5I4BPV=+E*%a# zj5Ns6Fof`3r|=o6{hnl8pNXf$$q?$g%}D*=bTD;ytfd!bJd1s`7yjc}S}MmVa!S8> zG)*#&Nh8?*V&UiWsE6d9baG8N{N_A5Rq{V@@?&Aee24bOlG{8VTVOA=&&Rs&1&`;j zjrBtDa~Kcd{}EV_M}r|W5*rI%&X6$)uV}fr8FZ#e2g%S z7&K@_%8Hc6+SU!Nme1pHO+Ss|SD0=nDx+Z-genPa61>YnzxqiT{ryV3)I5J4cNxD1 z*^F$%6r)S4xC+~%k%*y|hj8^gjm4Yaz4KHfZ#T98xIBsmikprt9J~@}q?TWUV8dM+ ziV-R9piv0cchC?7zv`edZZL2ilA>6#pGZ%j|o63LJ zOq88ZAIk1FgmKf`S843xAZS~q34km3DO~!~zFcFJBg3Q! zKajgihCf_Qp@Svv2R^AZ039l<)I=h%R%$}wOp8VbEtMLhG*!J!6DF~?a!pW%a4lfK zU-4URN!^;t8oS$p1G2hm{H?p9WJzUN)hd)i)M~zL)x28sL$tWT*dk!dSQ@3}_Z>)R z)(pY>OI3>|IGrPz8z{(7?wVfFWd zsHF!w3bcV*&Vq&l^q@zLpQb%trRA3n$WgUXP7KSG{{w=h6HF(xQQ}@<3xSS@B-9`t zJGL;`;zGi3&+@U7GCIecIiZb_W5VF92MN_;#)YK-epfMgR9VX_>?1<~zv6^q=|FS1%xop;l`~pjF(B(8fYhlLlGnbnoG294qsvbg;##~K2F3?l(zZd_#meuoubY4viRQ0OSv3eVIZPienBCk{c|Fu0XUhU>&D z4<5a$3)S+o5iY%|i;)TuP?kX=DfYnUH|nC}xzy%0+4@3NrM0Ylw!QoX*TcarTY~=> zf&)%()P>?9eSM=Y*jH${OfRcixvYZ!J4ZL@H|e4S1>4iXPUJSZENhc4OgdtIT^B7K z2?6|sX6Ht|uJcFBCpYP$@a_BcCf(8^13%?zY(wEt2o1oq*jHWsm2TRcaL%^*lvh+% z;HJk!lq|1XS&rco{J~JafDR$O>YaE!%Miy~jSUWMCWFB=O7Ek-w_JaA2A)S&RUadc zusR#eJVIjCld67Mu$CVzP;^xvh{s6g=lU=`XV`+Ftb~S2Vf9sgfW&@zRd1BoAt&!L y@*>3;(K_bU%&xVrDO+BVSXV70AtpF^MIRc-u{xW}a@>50{Qq)lTkGnUB>oRpjHqJ( diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index af9a42d0e5356dda579329707f03cc5fa4bc9d71..843a881b3ef8e69a7602cb5e9120bd5de26e2b66 100644 GIT binary patch delta 182 zcmaFR$M&?3tzinI5~KB2WkxdI)MWCwfftHL17zL3e^>Xsl)ioyoYe1 delta 173 zcmaFb$M&F)tzinI5~JBxWkx)gX!oDrVUJ5+!hwbdKQLe=Ei2GlN0o9HhXIEup$d?PSD@Sfh;gt(;^Us-)iB8 L%-?*?B0m@a%@!FZ delta 96 zcmX>)gX!oDrVUJ5+-9bRdgkU9mPSSvlN0o9HhXIEup$d?PSD@Sfh;gt(;^Us-)iB8 L%-?*?B0m@a%zGIo diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 6611db4aa..101e7571b 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"DRS Download Tools": [[10, null]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Gen3 Auth Helper": [[0, null]], "Gen3 File Class": [[1, null]], "Gen3 Index Class": [[3, null]], "Gen3 Jobs Class": [[4, null]], "Gen3 Metadata Class": [[5, null]], "Gen3 Object Class": [[6, null]], "Gen3 Query Class": [[7, null]], "Gen3 Submission Class": [[8, null]], "Gen3 Tools": [[9, null]], "Gen3 Workspace Storage": [[13, null]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Indexing Tools": [[11, null]], "Indices and tables": [[2, "indices-and-tables"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Metadata Tools": [[12, null]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, null]]}, "docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "indexentries": {"_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager", false]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods", false]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS", false]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create", false]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases", false]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record", false]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias", false]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases", false]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest", false]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get", false]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases", false]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record", false]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum", false]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page", false]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params", false]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest", false]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls", false]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd", false]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait", false]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update", false]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases", false]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record", false]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest", false]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider", false]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ", false]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create", false]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens", false]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children", false]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID", false]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url", false]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy", false]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create", false]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases", false]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank", false]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path", false]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job", false]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version", false]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list", false]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program", false]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project", false]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record", false]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time", false]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl", false]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR", false]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete", false]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias", false]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases", false]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids", false]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file", false]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations", false]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path", false]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node", false]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes", false]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object", false]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program", false]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project", false]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record", false]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record", false]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records", false]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download", false]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download", false]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download", false]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest", false]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single", false]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url", false]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable", false]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager", false]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus", false]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time", false]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint", false]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node", false]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record", false]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name", false]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name", false]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size", false]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size", false]], "gen3.tools": [[9, "module-gen3.tools", false]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download", false]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest", false]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest", false]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest", false]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest", false]], "gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth", false]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File", false]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index", false]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs", false]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata", false]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object", false]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query", false]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission", false]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage", false]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get", false]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get", false]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token", false]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts", false]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases", false]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records", false]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all", false]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node", false]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token", false]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema", false]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix", false]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths", false]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version", false]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output", false]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url", false]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs", false]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary", false]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest", false]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects", false]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record", false]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc", false]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records", false]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page", false]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats", false]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status", false]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls", false]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids", false]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version", false]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version", false]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version", false]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions", false]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params", false]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query", false]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID", false]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT", false]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT", false]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname", false]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest", false]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE", false]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy", false]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy", false]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy", false]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest", false]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object", false]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest", false]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs", false]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load", false]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest", false]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls", false]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path", false]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest", false]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS", false]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5", false]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum", false]], "module": [[9, "module-gen3.tools", false], [10, "module-gen3.tools.download.drs_download", false], [11, "module-gen3.tools.indexing.download_manifest", false], [11, "module-gen3.tools.indexing.index_manifest", false], [11, "module-gen3.tools.indexing.verify_manifest", false], [12, "module-gen3.tools.metadata.ingest_manifest", false]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id", false]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id", false]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type", false]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project", false]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint", false]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID", false]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query", false]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query", false]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query", false]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls", false]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download", false]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token", false]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects", false]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm", false]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path", false]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE", false]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time", false]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status", false]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file", false]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record", false]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl", false]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER", false]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update", false]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases", false]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank", false]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record", false]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time", false]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload", false]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file", false]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid", false]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url", false]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS", false]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access", false]]}, "objects": {"gen3": [[9, 3, 0, "-", "tools"]], "gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "terms": {"": [1, 4, 8, 10, 11], "0": [5, 7, 8, 11], "0084667": [], "013444": [], "0248635": [], "0791543": [], "0a80fada010c": 11, "0a80fada096c": 11, "0a80fada097c": 11, "0a80fada098c": 11, "0a80fada099c": 11, "1": [4, 5, 8, 10, 11], "10": [5, 7, 11], "100": [8, 11], "1125998": [], "11e9": 11, "120715": [], "1329331": [], "1468077": [], "165499": [], "1724170637": [], "1726237606": [], "1726237962": [], "1728059996": [], "1728059997": [], "1728061593": [], "1728061594": [], "1729113765": [], "1729629354": [], "1729629627": [], "1732401632": [], "1732401820": [], "1732401821": [], "1733254905": [], "1733263055": [], "1733431971": [], "1733932163": [], "1734031012": [], "1734031066": [], "1734031067": [], "1734474422": [], "1734474533": [], "1734474534": [], "1734644651": [11, 12], "2": [5, 11], "2023835": [], "2310405": [], "24": [11, 12], "244287": [], "2511687": [], "255e396f": 11, "256328": [], "284675": [], "3": [5, 11], "30": 8, "313695": [], "33": 5, "333": 5, "3410327": [], "343434344": 11, "363455714": 11, "3686683": [], "3697538": [], "3759608": [], "4": [5, 11], "4401503": [], "450c": 11, "4714": 8, "473d83400bc1bc9dc635e334fadd433c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "5": [0, 5], "50": 7, "5117643": [], "518752": [], "543434443": 11, "561932": 11, "5833347": [], "6129959": [], "6138082": [], "6146133": [], "653854": [], "6f90": 8, "7789228": 12, "7d3d8d2083b4": 11, "8366766": [], "8420": 8, "848236": [], "8493605": [], "8535511": [], "863009": [], "8712757": [], "9082778": [], "93d9af72": 11, "998105": [], "9a07": 11, "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "AND": 5, "Be": 1, "But": 5, "By": 11, "For": [1, 5, 6, 7, 8, 9, 11], "IF": 11, "If": [0, 1, 7, 11, 12], "In": 10, "It": 10, "NOT": 12, "OR": 5, "One": 7, "Such": 9, "THE": 11, "That": 3, "The": [0, 1, 2, 3, 5, 8, 10, 11], "There": 11, "These": 9, "To": 11, "Will": [4, 7, 10], "_get_acl_from_row": 11, "_get_authz_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_for_row": 12, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "_guid_typ": 12, "_manag": [2, 9, 10], "_query_for_associated_indexd_record_guid": 12, "_ssl": [3, 4, 5], "a5c6": 11, "ab167e49d25b488939b1ede42752458b": 3, "about": [2, 3], "abov": 11, "access": [0, 1, 3, 7, 10], "access_method": [2, 9, 10], "access_token": 0, "accesstoken": 0, "acl": [2, 3, 9, 11], "across": 11, "act": 0, "action": [9, 11], "actual": 11, "ad": 3, "add": [3, 5], "addit": [3, 5, 10, 11], "admin": [5, 11], "admin_endpoint_suffix": 5, "after": 10, "against": [3, 7, 8, 11, 12], "algorithm": 3, "alia": [3, 5], "alias": 5, "aliv": 7, "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "allow": [0, 6, 8, 10, 11, 12], "allowed_data_upload_bucket": 1, "along": 2, "alreadi": 9, "also": 1, "altern": [5, 11], "alwai": 5, "ammount": 12, "amount": [1, 9], "an": [0, 3, 6, 8, 10, 11], "ani": [0, 5, 10, 11], "anoth": 5, "api": [0, 5, 8, 11], "api_kei": 11, "appli": 7, "appropri": 13, "ar": [5, 7, 8, 9, 10, 11], "arbitrari": 0, "argument": [0, 13], "arrai": 8, "asc": 7, "assign": 9, "assist": 10, "associ": [3, 5], "assum": 11, "async": [3, 4, 5, 9, 11, 12], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_create_record": [2, 3], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_download_object_manifest": [2, 9, 11], "async_get": [2, 5], "async_get_alias": [2, 5], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_url": [2, 3], "async_query_urls_from_indexd": [2, 9, 12], "async_run_job_and_wait": [2, 4], "async_upd": [2, 5], "async_update_alias": [2, 5], "async_update_record": [2, 3], "async_verify_object_manifest": [2, 9, 11], "asynchron": [3, 4, 5], "asyncio": [11, 12], "asyncron": 5, "attach": [3, 5], "attempt": 11, "attribut": [10, 11], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "authbas": 0, "authent": 0, "author": 1, "authz": [0, 1, 2, 3, 9, 10, 11], "auto": [0, 2], "automat": 0, "avail": [1, 2, 10, 11], "az": 1, "b": [5, 11], "b0f1": 11, "bar": 10, "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "baseid": 3, "basic": [3, 11, 12], "batch_creat": [2, 5], "batch_siz": 8, "behalf": 0, "behavior": 11, "being": 3, "belong": 8, "below": 11, "blank": 3, "blob": [5, 7], "bodi": 3, "bool": [4, 5, 8, 10, 11, 12], "boolean": 3, "bownload": 10, "broad": 9, "broken": 9, "bucket": [1, 6], "bundl": 10, "byte": 10, "c": [5, 11], "cach": 10, "cache_hosts_wts_token": [2, 9, 10], "call": [10, 13], "can": [0, 3, 4, 8, 11, 12], "capabl": 9, "case": [0, 10], "categori": 9, "ccle": 8, "ccle_one_record": 8, "ccle_sample_nod": 8, "cdi": 7, "chang": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "child": 10, "children": [2, 9, 10], "chunk_siz": 8, "class": [0, 2, 10, 11, 13], "cli": 10, "client": [0, 3], "client_credenti": 0, "client_id": 0, "client_scop": 0, "client_secret": 0, "code": [2, 8], "column": [11, 12], "column_to_use_as_guid": [2, 9, 12], "columna": 11, "columnb": 11, "columnc": 11, "com": 7, "comma": 11, "command": [10, 11], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "commons_url": [2, 9, 10, 11, 12], "complet": [4, 11], "complex": 7, "concat": 11, "concurr": [11, 12], "configur": 1, "connect": 12, "consist": 3, "constructor": 0, "contain": [0, 2, 5, 8, 9, 10, 11, 12], "content": [3, 13], "content_created_d": 3, "content_updated_d": 3, "continu": 10, "control": 3, "conveni": 10, "copi": [2, 13], "coroutin": 11, "correspond": 3, "count": 3, "crdc": 0, "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_blank": [2, 3], "create_index_key_path": [2, 5], "create_job": [2, 4], "create_new_vers": [2, 3], "create_object_list": [2, 9, 10], "create_program": [2, 8], "create_project": [2, 8], "create_record": [2, 3], "created_tim": [2, 9, 10], "creation": [3, 11], "cred": 3, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "csv": [8, 11, 12], "curl": [0, 2], "current": [6, 8, 10], "current_dir": [2, 9, 11], "custom": 11, "d": 5, "d70b41b9": 8, "data": [0, 1, 3, 5, 7, 8, 10, 11], "data_spreadsheet": 8, "data_typ": 7, "data_upload_bucket": 1, "dataa": 11, "datab": 11, "databas": 5, "datacommon": 0, "datafil": 10, "datamanag": 10, "date": 3, "datetim": [1, 3, 10], "dbgap": 12, "dcf": 8, "def": 11, "default": [0, 1, 3, 7, 8, 11, 12], "defin": [5, 8, 10], "delai": 4, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_all_guid": [2, 9, 11], "delete_fil": [1, 2], "delete_file_loc": [1, 2, 6], "delete_index_key_path": [2, 5], "delete_nod": [2, 8], "delete_object": [2, 6], "delete_program": [2, 8], "delete_project": [2, 8], "delete_record": [2, 3, 8], "delete_unpacked_packag": 10, "delimet": [11, 12], "delimit": 11, "demograph": 8, "deprec": 1, "descript": [3, 5], "desir": 11, "dest_path": 13, "dest_urlstr": 13, "dest_w": 13, "dest_wskei": 13, "detail": [2, 7, 10], "determin": [10, 11, 12], "dev": 11, "dict": [3, 4, 5, 10, 11, 12], "dictionari": [3, 4, 5, 7, 8], "did": 3, "differ": 5, "direct": 0, "directori": [10, 11], "disabl": 10, "discoveri": 10, "disk": 13, "dispatch": 4, "dist_resolut": 3, "distribut": 3, "doc": [7, 10], "docstr": 2, "document": [1, 3], "doe": [0, 12], "domain": [11, 12], "done": 4, "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "download_files_in_drs_manifest": [2, 9, 10], "download_list": 10, "download_manifest": 11, "download_singl": [1, 2], "download_url": [2, 13], "downloadmanag": [2, 9, 10], "downloadstatu": [2, 9, 10], "dr": [2, 9], "drs_download": 10, "drs_hostnam": 10, "drsdownload": 10, "drsobjecttyp": 10, "e": [5, 10], "e043ab8b77b9": 8, "each": [3, 8, 10, 11], "effici": 9, "eg": 3, "either": 8, "elasticsearch": 7, "els": [0, 12], "elsewher": 12, "empti": [8, 11], "enabl": 11, "end": [5, 10], "end_tim": [2, 9, 10], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 10, 13], "entir": 8, "entri": [3, 11], "env": 0, "environ": 0, "equal": 7, "error": [10, 11, 12], "error_nam": 11, "etc": 8, "even": 11, "everi": [9, 11], "everyth": 11, "ex": [0, 11, 12], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "exclud": 3, "execut": [7, 8, 11], "exist": [1, 3, 5, 6, 9, 12], "expect": [5, 9, 11], "experi": 8, "expir": [0, 1], "expires_in": 1, "export": [8, 10], "export_nod": [2, 8], "export_record": [2, 8], "f1f8": 11, "factori": 10, "fail": [8, 10], "fals": [3, 5, 6, 10, 11], "featur": [1, 6], "fenc": [0, 1], "fetch": 0, "field": [3, 5, 7, 11, 12], "fieldnam": 11, "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "file_nam": [1, 2, 3, 9, 10, 11], "file_s": [2, 9, 10, 11], "file_st": 3, "fileformat": 8, "filenam": [0, 8, 10, 11, 12], "files": 10, "fill": 12, "filter": [5, 7], "filter_object": 7, "first": [7, 8], "flag": 11, "folder": 11, "follow": [0, 11], "forc": 11, "force_metadata_columns_even_if_empti": 11, "form": 13, "format": [3, 5, 8, 11], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "func_to_parse_row": [11, 12], "function": [2, 3, 4, 5, 9, 10, 11, 12], "g": 10, "gen3": [10, 11, 12], "gen3_api_kei": 0, "gen3_oidc_client_creds_secret": 0, "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "gen3fil": [1, 2], "gen3index": [2, 3], "gen3job": [2, 4, 10], "gen3metadata": [2, 5], "gen3object": [2, 6], "gen3queri": [2, 7], "gen3submiss": [2, 8], "gen3wsstorag": [2, 13], "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "get_access_token": [0, 2], "get_access_token_from_wt": [0, 2], "get_alias": [2, 5], "get_all_record": [2, 3], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_fresh_token": [2, 9, 10], "get_graphql_schema": [2, 8], "get_guid_from_fil": 12, "get_guids_prefix": [2, 3], "get_index_key_path": [2, 5], "get_latest_vers": [2, 3], "get_output": [2, 4], "get_presigned_url": [1, 2], "get_program": [2, 8], "get_project": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_statu": [2, 4], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "giangb": 11, "github": [2, 7], "give": 1, "given": [0, 3, 4, 5, 8, 10, 12, 13], "global": [4, 5], "good": 3, "grant": 0, "graph": 8, "graphql": [7, 8], "graphql_queri": [2, 7], "group": 3, "guid": [1, 2, 3, 5, 6, 9, 11, 12], "guid_exampl": 11, "guid_for_row": 12, "guid_from_fil": 12, "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "guppi": 7, "ha": [0, 11], "handl": [3, 10], "hardcod": 0, "has_vers": 3, "hash": [3, 11], "hash_typ": 3, "have": [5, 11], "header": 11, "healthi": [3, 4, 5], "help": 11, "helper": 2, "hit": 11, "host": 10, "hostnam": [2, 9, 10], "how": [8, 11], "howto": 10, "http": [0, 7, 11, 12], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "id": [0, 1, 3, 5, 10, 11], "idea": 3, "identifi": [3, 5, 9, 11], "idp": 0, "illustr": 11, "immut": 3, "implement": 0, "implic": 11, "import": 11, "includ": [0, 3], "indent": 10, "index": [0, 2, 5, 9], "index_manifest": 11, "index_object_manifest": [2, 9, 11], "indexd": [1, 3, 6, 10, 11, 12], "indexd_field": [11, 12], "indexd_record_page_s": [2, 9, 11], "indexed_file_object_guid": 12, "indic": [0, 11], "infil": 10, "info": [3, 11], "inform": [2, 3, 10], "ingest": [2, 9], "ingest_manifest": 12, "initi": [0, 10], "input": [4, 10, 11], "input_manifest": 11, "instal": [0, 2, 11], "instanc": [1, 3, 6, 7, 8, 9, 10], "instead": [1, 7, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "integ": [1, 3, 8], "intend": 0, "interact": [1, 3, 4, 5, 6, 8, 13], "interest": 10, "interpret": 0, "introspect": 8, "involv": 9, "is_healthi": [2, 3, 4, 5], "is_indexed_file_object": 12, "isn": 1, "issu": 0, "its": [1, 3], "job": 2, "job_id": 4, "job_input": 4, "job_nam": 4, "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "just": [5, 11, 12], "jwt": 0, "kei": [0, 3, 5, 13], "know": 11, "known": 10, "kwarg": [3, 4, 5], "l": [2, 13], "larg": 9, "last": 10, "latest": 3, "least": 3, "level": 6, "librari": 11, "like": [3, 5, 9, 11, 12], "limit": [1, 3, 5, 12], "linear": 4, "linux": 10, "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "list_job": [2, 4], "live": [11, 12], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "local": [0, 13], "locat": [1, 6], "lock": 12, "log": [8, 10, 11, 12], "logic": [5, 12], "loop": 11, "ls_path": [2, 13], "maco": 11, "made": 3, "mai": [0, 9, 11], "main": 10, "make": [9, 11], "manag": [1, 5, 10], "mani": [8, 11], "manifest": [2, 8, 9, 10, 11, 12], "manifest_1": 10, "manifest_fil": [11, 12], "manifest_file_delimit": [11, 12], "manifest_row_pars": [11, 12], "map": [0, 11], "mark": 8, "master": 7, "match": [3, 5, 12], "max": 5, "max_concurrent_request": [2, 9, 11, 12], "max_presigned_url_ttl": 1, "max_tri": 8, "maximum": [11, 12], "md": [5, 7, 10, 12], "md5": [2, 3, 9, 11], "md5_hash": 11, "md5sum": [2, 9, 10], "mean": 8, "mechan": 3, "merg": 5, "metadata": [2, 3, 6, 9, 11], "metadata_list": 5, "metadata_sourc": 12, "metadata_typ": 12, "metdata": 12, "method": [1, 7, 10], "minimum": 10, "minut": 0, "mode": 7, "modul": [2, 10, 11], "more": [2, 5, 7, 9, 10], "most": 9, "mostli": 2, "multipl": [8, 11], "must": [1, 5], "my_common": 10, "my_credenti": 10, "my_field": 7, "my_index": 7, "my_program": 7, "my_project": 7, "name": [3, 4, 8, 10, 11, 12, 13], "namespac": [0, 12], "necessari": [3, 5], "need": [3, 7, 10, 11], "nest": 5, "net": 11, "never": 0, "new": [0, 3], "node": 8, "node_nam": 8, "node_typ": 8, "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "note": [0, 3, 11, 12], "noth": [3, 6], "now": [1, 8], "num": 5, "num_process": 11, "num_total_fil": 11, "number": [3, 7, 8, 11, 12], "o": 0, "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "object_id": [1, 2, 9, 10], "object_list": 10, "object_typ": [2, 9, 10], "objectid": 10, "obtain": [0, 10], "occur": 10, "off": 5, "offset": [5, 7], "oidc": 0, "old": 3, "one": [3, 5, 10, 11], "onli": [3, 5, 7, 8, 10, 11], "open": [8, 10, 11], "open_project": [2, 8], "openid": 0, "opt": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "order": [0, 8], "ordered_node_list": 8, "org": 10, "otherwis": 10, "output": [4, 5, 11, 12], "output_dir": 10, "output_filenam": [11, 12], "overrid": [0, 11, 12], "overwrit": 5, "own": 0, "packag": 10, "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "pagin": 3, "parallel": 11, "param": [3, 5, 8, 10], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "pars": [10, 11, 12, 13], "parser": [11, 12], "particular": 0, "pass": [0, 7, 8, 10], "password": [11, 12], "path": [0, 1, 5, 10, 11, 13], "path_to_manifest": 11, "pattern": [3, 12], "pdcdatastor": 11, "pend": 10, "per": [11, 12], "peregrin": 8, "permiss": 10, "persist": 9, "phs0001": 11, "phs0002": 11, "pick": 1, "pla": 11, "place": 11, "planx": 11, "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "popul": [10, 12], "posit": [1, 7], "possibl": 10, "post": [0, 11], "pprint": [2, 9, 10], "prefix": 3, "presign": 1, "pretti": 10, "prev_guid": [2, 9, 11], "previou": [3, 11], "previous": 4, "print": [8, 10], "process": 11, "processed_fil": 11, "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "program": [8, 11], "progress": [8, 10], "project": [8, 11], "project_id": [7, 8], "protocol": 1, "provid": [0, 1, 3, 5, 7, 8, 12], "public": [3, 5], "put": 0, "py": 11, "python": [2, 9, 11], "python3": 11, "python_subprocess_command": 11, "queri": [1, 2, 3, 5, 8, 11, 12], "query_str": 7, "query_txt": [7, 8], "query_url": [2, 3], "quickstart": 2, "rather": 0, "raw": [7, 11], "raw_data_download": [2, 7], "rbac": 3, "read": [3, 5, 11], "readm": 2, "reason": 10, "record": [1, 3, 5, 7, 8, 11, 12], "refresh": [0, 10], "refresh_access_token": [0, 2], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "regist": 8, "regular": 7, "relat": 9, "remov": [1, 6, 11, 13], "replac": 11, "replace_url": 11, "repo": 2, "repres": [3, 5, 10], "represent": [1, 3], "request": [0, 1, 3, 5, 8, 11, 12], "requir": 10, "resolv": 10, "resolve_object": [2, 9, 10], "respect": 7, "respons": [0, 1, 3, 4, 5], "result": [1, 8, 10, 11], "retri": 8, "retriev": [1, 8, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "return_full_metadata": 5, "rev": 3, "revers": 8, "revis": 3, "right": 1, "rm": [2, 13], "rm_path": [2, 13], "root": [11, 12], "row": [7, 8, 11, 12], "row_offset": 8, "rtype": 3, "run": [8, 11], "s3": [1, 10, 11], "safe": 11, "same": [5, 11, 13], "sampl": [8, 10], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "save": 10, "save_directori": 10, "schema": 8, "scope": [0, 1], "screen": 8, "script": 2, "search": [0, 2, 3], "second": [1, 4], "secret": 0, "see": [7, 10, 11], "self": 10, "semaphon": 12, "semaphor": 12, "separ": [0, 11], "server": 10, "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "service_loc": [3, 4, 5], "session": 11, "set": [0, 1, 5, 10], "setup": 2, "sheepdog": 8, "should": [0, 8, 11], "show": 10, "show_progress": 10, "shown": 11, "sign": 1, "signpost": 3, "similar": 10, "simpl": 3, "simpli": 11, "sinc": 3, "singl": [1, 5, 8], "size": [2, 3, 9, 10, 11], "skip": 8, "sleep": 4, "so": 11, "some": [0, 2], "someth": 11, "sort": 7, "sort_field": 7, "sort_object": 7, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "space": [0, 11], "specif": [5, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "spreadsheet": 8, "src_path": 13, "src_urlstr": 13, "src_w": 13, "src_wskei": 13, "ssl": [3, 4, 5], "start": [4, 7, 8, 10], "start_tim": [2, 9, 10], "static": 10, "statu": [2, 4, 9, 10], "storag": [1, 2, 6], "store": [1, 3, 10], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "string": [0, 3, 5, 11, 13], "strip": 11, "sub": 8, "subject": [7, 8], "submiss": 2, "submit": [8, 11], "submit_additional_metadata_column": 11, "submit_fil": [2, 8], "submit_record": [2, 8], "submitter_id": 7, "success": 10, "successfulli": 10, "suffici": 3, "suppli": 3, "support": [0, 1, 5, 8, 11], "sure": 1, "synchron": 11, "syntax": 7, "system": [6, 7, 8, 9], "t": [1, 5, 11], "tab": 11, "task": 9, "temporari": 11, "test": 11, "test1": 11, "test2": 11, "test3": 11, "test4": 11, "test5": 11, "text": [1, 7, 8], "than": [0, 5], "thei": [0, 10], "them": [10, 11], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "those": 11, "thread": 11, "thread_num": 11, "threadcontrol": [2, 9, 11], "through": [8, 11], "tier": 7, "time": [1, 3, 8, 10, 11], "timestamp": 10, "tmp_folder": [2, 9, 11], "token": [0, 10], "tool": 2, "total": 11, "treat": [1, 5], "tree": 10, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "try": 0, "tsv": [8, 11, 12], "tupl": [0, 3, 11, 12], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "typic": 10, "uc": 7, "unaccess": 7, "under": [0, 8, 13], "uniqu": [1, 5], "unknown": 10, "unpack": 10, "unpack_packag": 10, "until": [4, 10], "up": [5, 9], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "update_blank": [2, 3], "update_record": [2, 3], "updated_tim": [2, 9, 10], "upload": [1, 2, 3, 8, 13], "upload_fil": [1, 2], "upload_file_to_guid": [1, 2], "upload_url": [2, 13], "url": [1, 2, 3, 9, 10, 11, 12, 13], "urls_metadata": 3, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "usag": 11, "use_agg_md": 5, "user": [0, 10, 12], "user_access": [2, 9, 10], "usual": 12, "utcnow": 1, "util": 9, "uuid": [1, 3, 8], "uuid1": 8, "uuid2": 8, "valid": [3, 7], "valu": [0, 1, 3, 5, 7, 10, 11], "value_from_indexd": 11, "value_from_manifest": 11, "variabl": [0, 7, 8], "variou": 2, "verbos": [7, 8], "verif": 11, "verifi": [2, 9], "verify_manifest": 11, "verify_object_manifest": 11, "version": [3, 4, 5], "vital_statu": 7, "w": 13, "wa": 0, "wai": 10, "wait": 4, "want": [0, 3, 8], "warn": 11, "we": [3, 11], "web": 0, "what": 5, "when": [0, 3, 7, 10, 12], "where": [0, 3, 5, 11, 12], "whether": [3, 4, 5, 8, 11, 12], "which": [8, 10], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "whose": 5, "within": [0, 2, 9], "without": [3, 5], "won": 5, "work": [0, 10], "workaround": 11, "worksheet": 8, "workspac": [0, 2], "wrapper": 10, "write": 11, "ws_urlstr": 13, "wskei": 13, "wss": 13, "wt": [0, 10], "x": 11, "xlsx": 8, "you": [0, 3, 8, 11], "your": 0}, "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "titleterms": {"": 2, "auth": 0, "class": [1, 3, 4, 5, 6, 7, 8], "document": 2, "download": [10, 11], "dr": 10, "file": 1, "gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "helper": 0, "index": [3, 11], "indic": 2, "ingest": 12, "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "sdk": 2, "storag": 13, "submiss": 8, "tabl": 2, "tool": [9, 10, 11, 12], "verifi": 11, "welcom": 2, "workspac": 13}}) \ No newline at end of file +Search.setIndex({"alltitles": {"DRS Download Tools": [[10, null]], "Download": [[11, "module-gen3.tools.indexing.download_manifest"]], "Gen3 Auth Helper": [[0, null]], "Gen3 File Class": [[1, null]], "Gen3 Index Class": [[3, null]], "Gen3 Jobs Class": [[4, null]], "Gen3 Metadata Class": [[5, null]], "Gen3 Object Class": [[6, null]], "Gen3 Query Class": [[7, null]], "Gen3 Submission Class": [[8, null]], "Gen3 Tools": [[9, null]], "Gen3 Workspace Storage": [[13, null]], "Index": [[11, "module-gen3.tools.indexing.index_manifest"]], "Indexing Tools": [[11, null]], "Indices and tables": [[2, "indices-and-tables"]], "Ingest": [[12, "module-gen3.tools.metadata.ingest_manifest"]], "Metadata Tools": [[12, null]], "Verify": [[11, "module-gen3.tools.indexing.verify_manifest"]], "Welcome to Gen3 SDK\u2019s documentation!": [[2, null]]}, "docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "indexentries": {"_manager (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable._manager", false]], "access_methods (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.access_methods", false]], "acls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ACLS", false]], "async_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create", false]], "async_create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_create_aliases", false]], "async_create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_create_record", false]], "async_delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_alias", false]], "async_delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_delete_aliases", false]], "async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.async_download_object_manifest", false]], "async_get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get", false]], "async_get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_get_aliases", false]], "async_get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_record", false]], "async_get_records_from_checksum() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_from_checksum", false]], "async_get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_records_on_page", false]], "async_get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_get_with_params", false]], "async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest", false]], "async_query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_query_urls", false]], "async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd", false]], "async_run_job_and_wait() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.async_run_job_and_wait", false]], "async_update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update", false]], "async_update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.async_update_aliases", false]], "async_update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.async_update_record", false]], "async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.async_verify_object_manifest", false]], "auth_provider (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.auth_provider", false]], "authz (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.AUTHZ", false]], "batch_create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.batch_create", false]], "cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens", false]], "children (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.children", false]], "column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID", false]], "commons_url (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.commons_url", false]], "copy() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.copy", false]], "create() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create", false]], "create_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_aliases", false]], "create_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_blank", false]], "create_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.create_index_key_path", false]], "create_job() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.create_job", false]], "create_new_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_new_version", false]], "create_object_list() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.create_object_list", false]], "create_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_program", false]], "create_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.create_project", false]], "create_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.create_record", false]], "created_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.created_time", false]], "curl() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.curl", false]], "current_dir (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.CURRENT_DIR", false]], "current_dir (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.CURRENT_DIR", false]], "delete() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete", false]], "delete_alias() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_alias", false]], "delete_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_aliases", false]], "delete_all_guids() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.delete_all_guids", false]], "delete_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file", false]], "delete_file_locations() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.delete_file_locations", false]], "delete_index_key_path() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.delete_index_key_path", false]], "delete_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_node", false]], "delete_nodes() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_nodes", false]], "delete_object() (gen3.object.gen3object method)": [[6, "gen3.object.Gen3Object.delete_object", false]], "delete_program() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_program", false]], "delete_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_project", false]], "delete_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.delete_record", false]], "delete_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_record", false]], "delete_records() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.delete_records", false]], "download() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.download", false]], "download() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.download", false]], "download() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download", false]], "download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.download_files_in_drs_manifest", false]], "download_single() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.download_single", false]], "download_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.download_url", false]], "downloadable (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Downloadable", false]], "downloadmanager (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadManager", false]], "downloadstatus (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.DownloadStatus", false]], "end_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.end_time", false]], "endpoint (gen3.metadata.gen3metadata attribute)": [[5, "gen3.metadata.Gen3Metadata.endpoint", false]], "export_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_node", false]], "export_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.export_record", false]], "file_name (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_name", false]], "file_name (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_name", false]], "file_size (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.file_size", false]], "file_size (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.file_size", false]], "gen3.tools": [[9, "module-gen3.tools", false]], "gen3.tools.download.drs_download": [[10, "module-gen3.tools.download.drs_download", false]], "gen3.tools.indexing.download_manifest": [[11, "module-gen3.tools.indexing.download_manifest", false]], "gen3.tools.indexing.index_manifest": [[11, "module-gen3.tools.indexing.index_manifest", false]], "gen3.tools.indexing.verify_manifest": [[11, "module-gen3.tools.indexing.verify_manifest", false]], "gen3.tools.metadata.ingest_manifest": [[12, "module-gen3.tools.metadata.ingest_manifest", false]], "gen3auth (class in gen3.auth)": [[0, "gen3.auth.Gen3Auth", false]], "gen3file (class in gen3.file)": [[1, "gen3.file.Gen3File", false]], "gen3index (class in gen3.index)": [[3, "gen3.index.Gen3Index", false]], "gen3jobs (class in gen3.jobs)": [[4, "gen3.jobs.Gen3Jobs", false]], "gen3metadata (class in gen3.metadata)": [[5, "gen3.metadata.Gen3Metadata", false]], "gen3object (class in gen3.object)": [[6, "gen3.object.Gen3Object", false]], "gen3query (class in gen3.query)": [[7, "gen3.query.Gen3Query", false]], "gen3submission (class in gen3.submission)": [[8, "gen3.submission.Gen3Submission", false]], "gen3wsstorage (class in gen3.wss)": [[13, "gen3.wss.Gen3WsStorage", false]], "get() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get", false]], "get() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get", false]], "get_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token", false]], "get_access_token_from_wts() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.get_access_token_from_wts", false]], "get_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_aliases", false]], "get_all_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_all_records", false]], "get_dictionary_all() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_all", false]], "get_dictionary_node() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_dictionary_node", false]], "get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.get_fresh_token", false]], "get_graphql_schema() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_graphql_schema", false]], "get_guids_prefix() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_guids_prefix", false]], "get_index_key_paths() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_index_key_paths", false]], "get_latest_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_latest_version", false]], "get_output() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_output", false]], "get_presigned_url() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.get_presigned_url", false]], "get_programs() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_programs", false]], "get_project_dictionary() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_dictionary", false]], "get_project_manifest() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_project_manifest", false]], "get_projects() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.get_projects", false]], "get_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record", false]], "get_record_doc() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_record_doc", false]], "get_records() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records", false]], "get_records_on_page() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_records_on_page", false]], "get_stats() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_stats", false]], "get_status() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_status", false]], "get_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_urls", false]], "get_valid_guids() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_valid_guids", false]], "get_version() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_version", false]], "get_version() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.get_version", false]], "get_version() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.get_version", false]], "get_versions() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_versions", false]], "get_with_params() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.get_with_params", false]], "graphql_query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.graphql_query", false]], "guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.GUID", false]], "guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT", false]], "guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT", false]], "hostname (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.hostname", false]], "index_object_manifest() (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.index_object_manifest", false]], "indexd_record_page_size (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE", false]], "is_healthy() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.is_healthy", false]], "is_healthy() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.is_healthy", false]], "is_healthy() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.is_healthy", false]], "list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_access_in_drs_manifest", false]], "list_drs_object() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_drs_object", false]], "list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.list_files_in_drs_manifest", false]], "list_jobs() (gen3.jobs.gen3jobs method)": [[4, "gen3.jobs.Gen3Jobs.list_jobs", false]], "load() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load", false]], "load_manifest() (gen3.tools.download.drs_download.manifest static method)": [[10, "gen3.tools.download.drs_download.Manifest.load_manifest", false]], "ls() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls", false]], "ls_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.ls_path", false]], "manifest (class in gen3.tools.download.drs_download)": [[10, "gen3.tools.download.drs_download.Manifest", false]], "max_concurrent_requests (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)": [[11, "gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS", false]], "max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)": [[12, "gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS", false]], "md5 (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.MD5", false]], "md5sum (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.md5sum", false]], "module": [[9, "module-gen3.tools", false], [10, "module-gen3.tools.download.drs_download", false], [11, "module-gen3.tools.indexing.download_manifest", false], [11, "module-gen3.tools.indexing.index_manifest", false], [11, "module-gen3.tools.indexing.verify_manifest", false], [12, "module-gen3.tools.metadata.ingest_manifest", false]], "object_id (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_id", false]], "object_id (gen3.tools.download.drs_download.manifest attribute)": [[10, "gen3.tools.download.drs_download.Manifest.object_id", false]], "object_type (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.object_type", false]], "open_project() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.open_project", false]], "pprint() (gen3.tools.download.drs_download.downloadable method)": [[10, "gen3.tools.download.drs_download.Downloadable.pprint", false]], "prev_guid (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.PREV_GUID", false]], "query() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.query", false]], "query() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.query", false]], "query() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.query", false]], "query_urls() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.query_urls", false]], "raw_data_download() (gen3.query.gen3query method)": [[7, "gen3.query.Gen3Query.raw_data_download", false]], "refresh_access_token() (gen3.auth.gen3auth method)": [[0, "gen3.auth.Gen3Auth.refresh_access_token", false]], "resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.resolve_objects", false]], "rm() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm", false]], "rm_path() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.rm_path", false]], "size (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.SIZE", false]], "start_time (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.start_time", false]], "status (gen3.tools.download.drs_download.downloadstatus attribute)": [[10, "gen3.tools.download.drs_download.DownloadStatus.status", false]], "submit_file() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_file", false]], "submit_record() (gen3.submission.gen3submission method)": [[8, "gen3.submission.Gen3Submission.submit_record", false]], "threadcontrol (class in gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.ThreadControl", false]], "tmp_folder (in module gen3.tools.indexing.download_manifest)": [[11, "gen3.tools.indexing.download_manifest.TMP_FOLDER", false]], "update() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update", false]], "update_aliases() (gen3.metadata.gen3metadata method)": [[5, "gen3.metadata.Gen3Metadata.update_aliases", false]], "update_blank() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_blank", false]], "update_record() (gen3.index.gen3index method)": [[3, "gen3.index.Gen3Index.update_record", false]], "updated_time (gen3.tools.download.drs_download.downloadable attribute)": [[10, "gen3.tools.download.drs_download.Downloadable.updated_time", false]], "upload() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload", false]], "upload_file() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file", false]], "upload_file_to_guid() (gen3.file.gen3file method)": [[1, "gen3.file.Gen3File.upload_file_to_guid", false]], "upload_url() (gen3.wss.gen3wsstorage method)": [[13, "gen3.wss.Gen3WsStorage.upload_url", false]], "urls (in module gen3.tools.indexing.index_manifest)": [[11, "gen3.tools.indexing.index_manifest.URLS", false]], "user_access() (gen3.tools.download.drs_download.downloadmanager method)": [[10, "gen3.tools.download.drs_download.DownloadManager.user_access", false]]}, "objects": {"gen3": [[9, 3, 0, "-", "tools"]], "gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "get_access_token_from_wts"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "download_single"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"], [1, 1, 1, "", "upload_file_to_guid"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_guids_prefix"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_valid_guids"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_create_aliases"], [5, 1, 1, "", "async_delete_alias"], [5, 1, 1, "", "async_delete_aliases"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_get_aliases"], [5, 1, 1, "", "async_update"], [5, 1, 1, "", "async_update_aliases"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_aliases"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_alias"], [5, 1, 1, "", "delete_aliases"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_aliases"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"], [5, 1, 1, "", "update_aliases"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "terms": {"": [1, 4, 8, 10, 11], "0": [5, 7, 8, 11], "0084667": [], "013444": [], "0248635": [], "0791543": [], "0a80fada010c": 11, "0a80fada096c": 11, "0a80fada097c": 11, "0a80fada098c": 11, "0a80fada099c": 11, "1": [4, 5, 8, 10, 11], "10": [5, 7, 11], "100": [8, 11], "1125998": [], "11e9": 11, "120715": [], "1329331": [], "1468077": [], "165499": [], "1724170637": [], "1726237606": [], "1726237962": [], "1728059996": [], "1728059997": [], "1728061593": [], "1728061594": [], "1729113765": [], "1729629354": [], "1729629627": [], "1732401632": [], "1732401820": [], "1732401821": [], "1733254905": [], "1733263055": [], "1733431971": [], "1733932163": [], "1734031012": [], "1734031066": [], "1734031067": [], "1734474422": [], "1734474533": [], "1734474534": [], "1734644651": [], "1734644883": [11, 12], "2": [5, 11], "2023835": [], "2310405": [], "24": [11, 12], "244287": [], "2511687": [], "255e396f": 11, "256328": [], "284675": [], "3": [5, 11], "30": 8, "313695": [], "33": 5, "333": 5, "3410327": [], "343434344": 11, "363455714": 11, "3686683": [], "3697538": [], "3759608": [], "4": [5, 11], "4401503": [], "450c": 11, "4714": 8, "473d83400bc1bc9dc635e334fadd433c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "5": [0, 5], "50": 7, "5117643": [], "518752": [], "543434443": 11, "5574236": 11, "561932": [], "5833347": [], "6129959": [], "6138082": [], "6146133": [], "653854": [], "6f90": 8, "7789228": [], "7d3d8d2083b4": 11, "8167365": 12, "8366766": [], "8420": 8, "848236": [], "8493605": [], "8535511": [], "863009": [], "8712757": [], "9082778": [], "93d9af72": 11, "998105": [], "9a07": 11, "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "AND": 5, "Be": 1, "But": 5, "By": 11, "For": [1, 5, 6, 7, 8, 9, 11], "IF": 11, "If": [0, 1, 7, 11, 12], "In": 10, "It": 10, "NOT": 12, "OR": 5, "One": 7, "Such": 9, "THE": 11, "That": 3, "The": [0, 1, 2, 3, 5, 8, 10, 11], "There": 11, "These": 9, "To": 11, "Will": [4, 7, 10], "_get_acl_from_row": 11, "_get_authz_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_for_row": 12, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "_guid_typ": 12, "_manag": [2, 9, 10], "_query_for_associated_indexd_record_guid": 12, "_ssl": [3, 4, 5], "a5c6": 11, "ab167e49d25b488939b1ede42752458b": 3, "about": [2, 3], "abov": 11, "access": [0, 1, 3, 7, 10], "access_method": [2, 9, 10], "access_token": 0, "accesstoken": 0, "acl": [2, 3, 9, 11], "across": 11, "act": 0, "action": [9, 11], "actual": 11, "ad": 3, "add": [3, 5], "addit": [3, 5, 10, 11], "admin": [5, 11], "admin_endpoint_suffix": 5, "after": 10, "against": [3, 7, 8, 11, 12], "algorithm": 3, "alia": [3, 5], "alias": 5, "aliv": 7, "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "allow": [0, 6, 8, 10, 11, 12], "allowed_data_upload_bucket": 1, "along": 2, "alreadi": 9, "also": 1, "altern": [5, 11], "alwai": 5, "ammount": 12, "amount": [1, 9], "an": [0, 3, 6, 8, 10, 11], "ani": [0, 5, 10, 11], "anoth": 5, "api": [0, 5, 8, 11], "api_kei": 11, "appli": 7, "appropri": 13, "ar": [5, 7, 8, 9, 10, 11], "arbitrari": 0, "argument": [0, 13], "arrai": 8, "asc": 7, "assign": 9, "assist": 10, "associ": [3, 5], "assum": 11, "async": [3, 4, 5, 9, 11, 12], "async_cr": [2, 5], "async_create_alias": [2, 5], "async_create_record": [2, 3], "async_delete_alia": [2, 5], "async_delete_alias": [2, 5], "async_download_object_manifest": [2, 9, 11], "async_get": [2, 5], "async_get_alias": [2, 5], "async_get_record": [2, 3], "async_get_records_from_checksum": [2, 3], "async_get_records_on_pag": [2, 3], "async_get_with_param": [2, 3], "async_ingest_metadata_manifest": [2, 9, 12], "async_query_url": [2, 3], "async_query_urls_from_indexd": [2, 9, 12], "async_run_job_and_wait": [2, 4], "async_upd": [2, 5], "async_update_alias": [2, 5], "async_update_record": [2, 3], "async_verify_object_manifest": [2, 9, 11], "asynchron": [3, 4, 5], "asyncio": [11, 12], "asyncron": 5, "attach": [3, 5], "attempt": 11, "attribut": [10, 11], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "auth_provid": [1, 2, 3, 4, 5, 6, 7, 8, 13], "authbas": 0, "authent": 0, "author": 1, "authz": [0, 1, 2, 3, 9, 10, 11], "auto": [0, 2], "automat": 0, "avail": [1, 2, 10, 11], "az": 1, "b": [5, 11], "b0f1": 11, "bar": 10, "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "baseid": 3, "basic": [3, 11, 12], "batch_creat": [2, 5], "batch_siz": 8, "behalf": 0, "behavior": 11, "being": 3, "belong": 8, "below": 11, "blank": 3, "blob": [5, 7], "bodi": 3, "bool": [4, 5, 8, 10, 11, 12], "boolean": 3, "bownload": 10, "broad": 9, "broken": 9, "bucket": [1, 6], "bundl": 10, "byte": 10, "c": [5, 11], "cach": 10, "cache_hosts_wts_token": [2, 9, 10], "call": [10, 13], "can": [0, 3, 4, 8, 11, 12], "capabl": 9, "case": [0, 10], "categori": 9, "ccle": 8, "ccle_one_record": 8, "ccle_sample_nod": 8, "cdi": 7, "chang": [3, 11], "checksum": [3, 10], "checksum_typ": 3, "child": 10, "children": [2, 9, 10], "chunk_siz": 8, "class": [0, 2, 10, 11, 13], "cli": 10, "client": [0, 3], "client_credenti": 0, "client_id": 0, "client_scop": 0, "client_secret": 0, "code": [2, 8], "column": [11, 12], "column_to_use_as_guid": [2, 9, 12], "columna": 11, "columnb": 11, "columnc": 11, "com": 7, "comma": 11, "command": [10, 11], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "commons_url": [2, 9, 10, 11, 12], "complet": [4, 11], "complex": 7, "concat": 11, "concurr": [11, 12], "configur": 1, "connect": 12, "consist": 3, "constructor": 0, "contain": [0, 2, 5, 8, 9, 10, 11, 12], "content": [3, 13], "content_created_d": 3, "content_updated_d": 3, "continu": 10, "control": 3, "conveni": 10, "copi": [2, 13], "coroutin": 11, "correspond": 3, "count": 3, "crdc": 0, "creat": [2, 3, 4, 5, 6, 8, 10, 11], "create_alias": [2, 5], "create_blank": [2, 3], "create_index_key_path": [2, 5], "create_job": [2, 4], "create_new_vers": [2, 3], "create_object_list": [2, 9, 10], "create_program": [2, 8], "create_project": [2, 8], "create_record": [2, 3], "created_tim": [2, 9, 10], "creation": [3, 11], "cred": 3, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "csv": [8, 11, 12], "curl": [0, 2], "current": [6, 8, 10], "current_dir": [2, 9, 11], "custom": 11, "d": 5, "d70b41b9": 8, "data": [0, 1, 3, 5, 7, 8, 10, 11], "data_spreadsheet": 8, "data_typ": 7, "data_upload_bucket": 1, "dataa": 11, "datab": 11, "databas": 5, "datacommon": 0, "datafil": 10, "datamanag": 10, "date": 3, "datetim": [1, 3, 10], "dbgap": 12, "dcf": 8, "def": 11, "default": [0, 1, 3, 7, 8, 11, 12], "defin": [5, 8, 10], "delai": 4, "delet": [0, 1, 2, 3, 5, 6, 8, 10, 11], "delete_alia": [2, 5], "delete_alias": [2, 5], "delete_all_guid": [2, 9, 11], "delete_fil": [1, 2], "delete_file_loc": [1, 2, 6], "delete_index_key_path": [2, 5], "delete_nod": [2, 8], "delete_object": [2, 6], "delete_program": [2, 8], "delete_project": [2, 8], "delete_record": [2, 3, 8], "delete_unpacked_packag": 10, "delimet": [11, 12], "delimit": 11, "demograph": 8, "deprec": 1, "descript": [3, 5], "desir": 11, "dest_path": 13, "dest_urlstr": 13, "dest_w": 13, "dest_wskei": 13, "detail": [2, 7, 10], "determin": [10, 11, 12], "dev": 11, "dict": [3, 4, 5, 10, 11, 12], "dictionari": [3, 4, 5, 7, 8], "did": 3, "differ": 5, "direct": 0, "directori": [10, 11], "disabl": 10, "discoveri": 10, "disk": 13, "dispatch": 4, "dist_resolut": 3, "distribut": 3, "doc": [7, 10], "docstr": 2, "document": [1, 3], "doe": [0, 12], "domain": [11, 12], "done": 4, "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "download_files_in_drs_manifest": [2, 9, 10], "download_list": 10, "download_manifest": 11, "download_singl": [1, 2], "download_url": [2, 13], "downloadmanag": [2, 9, 10], "downloadstatu": [2, 9, 10], "dr": [2, 9], "drs_download": 10, "drs_hostnam": 10, "drsdownload": 10, "drsobjecttyp": 10, "e": [5, 10], "e043ab8b77b9": 8, "each": [3, 8, 10, 11], "effici": 9, "eg": 3, "either": 8, "elasticsearch": 7, "els": [0, 12], "elsewher": 12, "empti": [8, 11], "enabl": 11, "end": [5, 10], "end_tim": [2, 9, 10], "endpoint": [0, 1, 2, 3, 4, 5, 7, 8, 10, 13], "entir": 8, "entri": [3, 11], "env": 0, "environ": 0, "equal": 7, "error": [10, 11, 12], "error_nam": 11, "etc": 8, "even": 11, "everi": [9, 11], "everyth": 11, "ex": [0, 11, 12], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "exclud": 3, "execut": [7, 8, 11], "exist": [1, 3, 5, 6, 9, 12], "expect": [5, 9, 11], "experi": 8, "expir": [0, 1], "expires_in": 1, "export": [8, 10], "export_nod": [2, 8], "export_record": [2, 8], "f1f8": 11, "factori": 10, "fail": [8, 10], "fals": [3, 5, 6, 10, 11], "featur": [1, 6], "fenc": [0, 1], "fetch": 0, "field": [3, 5, 7, 11, 12], "fieldnam": 11, "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "file_nam": [1, 2, 3, 9, 10, 11], "file_s": [2, 9, 10, 11], "file_st": 3, "fileformat": 8, "filenam": [0, 8, 10, 11, 12], "files": 10, "fill": 12, "filter": [5, 7], "filter_object": 7, "first": [7, 8], "flag": 11, "folder": 11, "follow": [0, 11], "forc": 11, "force_metadata_columns_even_if_empti": 11, "form": 13, "format": [3, 5, 8, 11], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "func_to_parse_row": [11, 12], "function": [2, 3, 4, 5, 9, 10, 11, 12], "g": 10, "gen3": [10, 11, 12], "gen3_api_kei": 0, "gen3_oidc_client_creds_secret": 0, "gen3auth": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "gen3fil": [1, 2], "gen3index": [2, 3], "gen3job": [2, 4, 10], "gen3metadata": [2, 5], "gen3object": [2, 6], "gen3queri": [2, 7], "gen3submiss": [2, 8], "gen3wsstorag": [2, 13], "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "get": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12, 13], "get_access_token": [0, 2], "get_access_token_from_wt": [0, 2], "get_alias": [2, 5], "get_all_record": [2, 3], "get_dictionary_al": [2, 8], "get_dictionary_nod": [2, 8], "get_fresh_token": [2, 9, 10], "get_graphql_schema": [2, 8], "get_guid_from_fil": 12, "get_guids_prefix": [2, 3], "get_index_key_path": [2, 5], "get_latest_vers": [2, 3], "get_output": [2, 4], "get_presigned_url": [1, 2], "get_program": [2, 8], "get_project": [2, 8], "get_project_dictionari": [2, 8], "get_project_manifest": [2, 8], "get_record": [2, 3], "get_record_doc": [2, 3], "get_records_on_pag": [2, 3], "get_stat": [2, 3], "get_statu": [2, 4], "get_url": [2, 3], "get_valid_guid": [2, 3], "get_vers": [2, 3, 4, 5], "get_with_param": [2, 3], "giangb": 11, "github": [2, 7], "give": 1, "given": [0, 3, 4, 5, 8, 10, 12, 13], "global": [4, 5], "good": 3, "grant": 0, "graph": 8, "graphql": [7, 8], "graphql_queri": [2, 7], "group": 3, "guid": [1, 2, 3, 5, 6, 9, 11, 12], "guid_exampl": 11, "guid_for_row": 12, "guid_from_fil": 12, "guid_type_for_indexed_file_object": [2, 9, 12], "guid_type_for_non_indexed_file_object": [2, 9, 12], "guppi": 7, "ha": [0, 11], "handl": [3, 10], "hardcod": 0, "has_vers": 3, "hash": [3, 11], "hash_typ": 3, "have": [5, 11], "header": 11, "healthi": [3, 4, 5], "help": 11, "helper": 2, "hit": 11, "host": 10, "hostnam": [2, 9, 10], "how": [8, 11], "howto": 10, "http": [0, 7, 11, 12], "i": [0, 1, 2, 3, 4, 5, 8, 10, 11, 12], "id": [0, 1, 3, 5, 10, 11], "idea": 3, "identifi": [3, 5, 9, 11], "idp": 0, "illustr": 11, "immut": 3, "implement": 0, "implic": 11, "import": 11, "includ": [0, 3], "indent": 10, "index": [0, 2, 5, 9], "index_manifest": 11, "index_object_manifest": [2, 9, 11], "indexd": [1, 3, 6, 10, 11, 12], "indexd_field": [11, 12], "indexd_record_page_s": [2, 9, 11], "indexed_file_object_guid": 12, "indic": [0, 11], "infil": 10, "info": [3, 11], "inform": [2, 3, 10], "ingest": [2, 9], "ingest_manifest": 12, "initi": [0, 10], "input": [4, 10, 11], "input_manifest": 11, "instal": [0, 2, 11], "instanc": [1, 3, 6, 7, 8, 9, 10], "instead": [1, 7, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "integ": [1, 3, 8], "intend": 0, "interact": [1, 3, 4, 5, 6, 8, 13], "interest": 10, "interpret": 0, "introspect": 8, "involv": 9, "is_healthi": [2, 3, 4, 5], "is_indexed_file_object": 12, "isn": 1, "issu": 0, "its": [1, 3], "job": 2, "job_id": 4, "job_input": 4, "job_nam": 4, "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "just": [5, 11, 12], "jwt": 0, "kei": [0, 3, 5, 13], "know": 11, "known": 10, "kwarg": [3, 4, 5], "l": [2, 13], "larg": 9, "last": 10, "latest": 3, "least": 3, "level": 6, "librari": 11, "like": [3, 5, 9, 11, 12], "limit": [1, 3, 5, 12], "linear": 4, "linux": 10, "list": [0, 1, 3, 4, 5, 7, 8, 10, 11, 13], "list_access_in_drs_manifest": [2, 9, 10], "list_drs_object": [2, 9, 10], "list_files_in_drs_manifest": [2, 9, 10], "list_job": [2, 4], "live": [11, 12], "load": [2, 9, 10], "load_manifest": [2, 9, 10], "local": [0, 13], "locat": [1, 6], "lock": 12, "log": [8, 10, 11, 12], "logic": [5, 12], "loop": 11, "ls_path": [2, 13], "maco": 11, "made": 3, "mai": [0, 9, 11], "main": 10, "make": [9, 11], "manag": [1, 5, 10], "mani": [8, 11], "manifest": [2, 8, 9, 10, 11, 12], "manifest_1": 10, "manifest_fil": [11, 12], "manifest_file_delimit": [11, 12], "manifest_row_pars": [11, 12], "map": [0, 11], "mark": 8, "master": 7, "match": [3, 5, 12], "max": 5, "max_concurrent_request": [2, 9, 11, 12], "max_presigned_url_ttl": 1, "max_tri": 8, "maximum": [11, 12], "md": [5, 7, 10, 12], "md5": [2, 3, 9, 11], "md5_hash": 11, "md5sum": [2, 9, 10], "mean": 8, "mechan": 3, "merg": 5, "metadata": [2, 3, 6, 9, 11], "metadata_list": 5, "metadata_sourc": 12, "metadata_typ": 12, "metdata": 12, "method": [1, 7, 10], "minimum": 10, "minut": 0, "mode": 7, "modul": [2, 10, 11], "more": [2, 5, 7, 9, 10], "most": 9, "mostli": 2, "multipl": [8, 11], "must": [1, 5], "my_common": 10, "my_credenti": 10, "my_field": 7, "my_index": 7, "my_program": 7, "my_project": 7, "name": [3, 4, 8, 10, 11, 12, 13], "namespac": [0, 12], "necessari": [3, 5], "need": [3, 7, 10, 11], "nest": 5, "net": 11, "never": 0, "new": [0, 3], "node": 8, "node_nam": 8, "node_typ": 8, "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "note": [0, 3, 11, 12], "noth": [3, 6], "now": [1, 8], "num": 5, "num_process": 11, "num_total_fil": 11, "number": [3, 7, 8, 11, 12], "o": 0, "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "object_id": [1, 2, 9, 10], "object_list": 10, "object_typ": [2, 9, 10], "objectid": 10, "obtain": [0, 10], "occur": 10, "off": 5, "offset": [5, 7], "oidc": 0, "old": 3, "one": [3, 5, 10, 11], "onli": [3, 5, 7, 8, 10, 11], "open": [8, 10, 11], "open_project": [2, 8], "openid": 0, "opt": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "order": [0, 8], "ordered_node_list": 8, "org": 10, "otherwis": 10, "output": [4, 5, 11, 12], "output_dir": 10, "output_filenam": [11, 12], "overrid": [0, 11, 12], "overwrit": 5, "own": 0, "packag": 10, "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "pagin": 3, "parallel": 11, "param": [3, 5, 8, 10], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "pars": [10, 11, 12, 13], "parser": [11, 12], "particular": 0, "pass": [0, 7, 8, 10], "password": [11, 12], "path": [0, 1, 5, 10, 11, 13], "path_to_manifest": 11, "pattern": [3, 12], "pdcdatastor": 11, "pend": 10, "per": [11, 12], "peregrin": 8, "permiss": 10, "persist": 9, "phs0001": 11, "phs0002": 11, "pick": 1, "pla": 11, "place": 11, "planx": 11, "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "popul": [10, 12], "posit": [1, 7], "possibl": 10, "post": [0, 11], "pprint": [2, 9, 10], "prefix": 3, "presign": 1, "pretti": 10, "prev_guid": [2, 9, 11], "previou": [3, 11], "previous": 4, "print": [8, 10], "process": 11, "processed_fil": 11, "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "program": [8, 11], "progress": [8, 10], "project": [8, 11], "project_id": [7, 8], "protocol": 1, "provid": [0, 1, 3, 5, 7, 8, 12], "public": [3, 5], "put": 0, "py": 11, "python": [2, 9, 11], "python3": 11, "python_subprocess_command": 11, "queri": [1, 2, 3, 5, 8, 11, 12], "query_str": 7, "query_txt": [7, 8], "query_url": [2, 3], "quickstart": 2, "rather": 0, "raw": [7, 11], "raw_data_download": [2, 7], "rbac": 3, "read": [3, 5, 11], "readm": 2, "reason": 10, "record": [1, 3, 5, 7, 8, 11, 12], "refresh": [0, 10], "refresh_access_token": [0, 2], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "regist": 8, "regular": 7, "relat": 9, "remov": [1, 6, 11, 13], "replac": 11, "replace_url": 11, "repo": 2, "repres": [3, 5, 10], "represent": [1, 3], "request": [0, 1, 3, 5, 8, 11, 12], "requir": 10, "resolv": 10, "resolve_object": [2, 9, 10], "respect": 7, "respons": [0, 1, 3, 4, 5], "result": [1, 8, 10, 11], "retri": 8, "retriev": [1, 8, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "return_full_metadata": 5, "rev": 3, "revers": 8, "revis": 3, "right": 1, "rm": [2, 13], "rm_path": [2, 13], "root": [11, 12], "row": [7, 8, 11, 12], "row_offset": 8, "rtype": 3, "run": [8, 11], "s3": [1, 10, 11], "safe": 11, "same": [5, 11, 13], "sampl": [8, 10], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "save": 10, "save_directori": 10, "schema": 8, "scope": [0, 1], "screen": 8, "script": 2, "search": [0, 2, 3], "second": [1, 4], "secret": 0, "see": [7, 10, 11], "self": 10, "semaphon": 12, "semaphor": 12, "separ": [0, 11], "server": 10, "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "service_loc": [3, 4, 5], "session": 11, "set": [0, 1, 5, 10], "setup": 2, "sheepdog": 8, "should": [0, 8, 11], "show": 10, "show_progress": 10, "shown": 11, "sign": 1, "signpost": 3, "similar": 10, "simpl": 3, "simpli": 11, "sinc": 3, "singl": [1, 5, 8], "size": [2, 3, 9, 10, 11], "skip": 8, "sleep": 4, "so": 11, "some": [0, 2], "someth": 11, "sort": 7, "sort_field": 7, "sort_object": 7, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "space": [0, 11], "specif": [5, 8, 11, 12], "specifi": [0, 1, 3, 11, 13], "spreadsheet": 8, "src_path": 13, "src_urlstr": 13, "src_w": 13, "src_wskei": 13, "ssl": [3, 4, 5], "start": [4, 7, 8, 10], "start_tim": [2, 9, 10], "static": 10, "statu": [2, 4, 9, 10], "storag": [1, 2, 6], "store": [1, 3, 10], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "string": [0, 3, 5, 11, 13], "strip": 11, "sub": 8, "subject": [7, 8], "submiss": 2, "submit": [8, 11], "submit_additional_metadata_column": 11, "submit_fil": [2, 8], "submit_record": [2, 8], "submitter_id": 7, "success": 10, "successfulli": 10, "suffici": 3, "suppli": 3, "support": [0, 1, 5, 8, 11], "sure": 1, "synchron": 11, "syntax": 7, "system": [6, 7, 8, 9], "t": [1, 5, 11], "tab": 11, "task": 9, "temporari": 11, "test": 11, "test1": 11, "test2": 11, "test3": 11, "test4": 11, "test5": 11, "text": [1, 7, 8], "than": [0, 5], "thei": [0, 10], "them": [10, 11], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "those": 11, "thread": 11, "thread_num": 11, "threadcontrol": [2, 9, 11], "through": [8, 11], "tier": 7, "time": [1, 3, 8, 10, 11], "timestamp": 10, "tmp_folder": [2, 9, 11], "token": [0, 10], "tool": 2, "total": 11, "treat": [1, 5], "tree": 10, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "try": 0, "tsv": [8, 11, 12], "tupl": [0, 3, 11, 12], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "typic": 10, "uc": 7, "unaccess": 7, "under": [0, 8, 13], "uniqu": [1, 5], "unknown": 10, "unpack": 10, "unpack_packag": 10, "until": [4, 10], "up": [5, 9], "updat": [2, 3, 5, 10, 11], "update_alias": [2, 5], "update_blank": [2, 3], "update_record": [2, 3], "updated_tim": [2, 9, 10], "upload": [1, 2, 3, 8, 13], "upload_fil": [1, 2], "upload_file_to_guid": [1, 2], "upload_url": [2, 13], "url": [1, 2, 3, 9, 10, 11, 12, 13], "urls_metadata": 3, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "usag": 11, "use_agg_md": 5, "user": [0, 10, 12], "user_access": [2, 9, 10], "usual": 12, "utcnow": 1, "util": 9, "uuid": [1, 3, 8], "uuid1": 8, "uuid2": 8, "valid": [3, 7], "valu": [0, 1, 3, 5, 7, 10, 11], "value_from_indexd": 11, "value_from_manifest": 11, "variabl": [0, 7, 8], "variou": 2, "verbos": [7, 8], "verif": 11, "verifi": [2, 9], "verify_manifest": 11, "verify_object_manifest": 11, "version": [3, 4, 5], "vital_statu": 7, "w": 13, "wa": 0, "wai": 10, "wait": 4, "want": [0, 3, 8], "warn": 11, "we": [3, 11], "web": 0, "what": 5, "when": [0, 3, 7, 10, 12], "where": [0, 3, 5, 11, 12], "whether": [3, 4, 5, 8, 11, 12], "which": [8, 10], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "whose": 5, "within": [0, 2, 9], "without": [3, 5], "won": 5, "work": [0, 10], "workaround": 11, "worksheet": 8, "workspac": [0, 2], "wrapper": 10, "write": 11, "ws_urlstr": 13, "wskei": 13, "wss": 13, "wt": [0, 10], "x": 11, "xlsx": 8, "you": [0, 3, 8, 11], "your": 0}, "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "titleterms": {"": 2, "auth": 0, "class": [1, 3, 4, 5, 6, 7, 8], "document": 2, "download": [10, 11], "dr": 10, "file": 1, "gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "helper": 0, "index": [3, 11], "indic": 2, "ingest": 12, "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "sdk": 2, "storag": 13, "submiss": 8, "tabl": 2, "tool": [9, 10, 11, 12], "verifi": 11, "welcom": 2, "workspac": 13}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index d67d1b2df..241cfc031 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -381,7 +381,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1734644651.561932.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1734644883.5574236.log')[source]

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index be7bef327..527771b39 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -102,7 +102,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1734644651.7789228.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1734644883.8167365.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters: