diff --git a/.secrets.baseline b/.secrets.baseline index d49240ea8..fa8b5f236 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "(docs\\/_build|poetry.lock)|^.secrets.baseline$", "lines": null }, - "generated_at": "2022-06-07T20:32:38Z", + "generated_at": "2022-06-10T15:20:47Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -62,19 +62,19 @@ { "hashed_secret": "7c35c215b326b9463b669b657c1ff9873ff53d9a", "is_verified": false, - "line_number": 202, + "line_number": 203, "type": "Hex High Entropy String" }, { "hashed_secret": "0d515eaf06062d52e8c80abb4d3b713a65396d30", "is_verified": false, - "line_number": 207, + "line_number": 208, "type": "Hex High Entropy String" }, { "hashed_secret": "b4cff7c2af45cdfe66195ec574a7b8832f8621ea", "is_verified": false, - "line_number": 214, + "line_number": 215, "type": "Hex High Entropy String" } ], @@ -350,49 +350,49 @@ { "hashed_secret": "96c9184fb19c9c1618ccf44d141f8029a739891c", "is_verified": false, - "line_number": 426, + "line_number": 424, "type": "Hex High Entropy String" }, { "hashed_secret": "e1da93616713812cb50e0ac845b1e9e305d949f1", "is_verified": false, - "line_number": 562, + "line_number": 559, "type": "Hex High Entropy String" }, { "hashed_secret": "47f42f4c34fddab383b817e689dc0fb75af81266", "is_verified": false, - "line_number": 586, + "line_number": 583, "type": "Hex High Entropy String" }, { "hashed_secret": "300d95dd5d30ab6928ffda6c08c6a129a23e5b39", "is_verified": false, - "line_number": 610, + "line_number": 607, "type": "Hex High Entropy String" }, { "hashed_secret": "f9e664db75c7f23a299b0b055c10e08d47073e93", "is_verified": false, - "line_number": 678, + "line_number": 675, "type": "Hex High Entropy String" }, { "hashed_secret": "7c35c215b326b9463b669b657c1ff9873ff53d9a", "is_verified": false, - "line_number": 819, + "line_number": 816, "type": "Hex High Entropy String" }, { "hashed_secret": "0d515eaf06062d52e8c80abb4d3b713a65396d30", "is_verified": false, - "line_number": 823, + "line_number": 820, "type": "Hex High Entropy String" }, { "hashed_secret": "b4cff7c2af45cdfe66195ec574a7b8832f8621ea", "is_verified": false, - "line_number": 828, + "line_number": 825, "type": "Hex High Entropy String" } ], diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index 022ceb53f..271deea41 100644 Binary files a/docs/_build/doctrees/environment.pickle and b/docs/_build/doctrees/environment.pickle differ diff --git a/docs/_build/doctrees/tools/indexing.doctree b/docs/_build/doctrees/tools/indexing.doctree index aefbfd1a1..6c48b29a0 100644 Binary files a/docs/_build/doctrees/tools/indexing.doctree and b/docs/_build/doctrees/tools/indexing.doctree differ diff --git a/docs/_build/doctrees/tools/metadata.doctree b/docs/_build/doctrees/tools/metadata.doctree index 616a9c69c..2de6d9326 100644 Binary files a/docs/_build/doctrees/tools/metadata.doctree and b/docs/_build/doctrees/tools/metadata.doctree differ diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js index 8ecf6599e..2f9169080 100644 --- a/docs/_build/html/searchindex.js +++ b/docs/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "initi": [0, 10], "follow": [0, 11], "curl": 0, "data": [0, 1, 3, 5, 7, 8, 10, 11], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "user": [0, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 3, 10, 11], "map": [0, 11], "get": [0, 1, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 3, 5, 6, 8, 10, 11], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": 0, "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "refresh_access_token": 0, "new": [0, 3], "gen3fil": 1, "auth_provid": [1, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": 1, "guid": [1, 3, 5, 6, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "id": [1, 3, 10, 11], "text": [1, 7, 8], "result": [1, 8, 10], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "get_presigned_url": 1, "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": 1, "file_nam": [1, 3, 10, 11], "expires_in": 1, "upload": [1, 3, 8, 13], "list": [1, 3, 4, 5, 7, 8, 10, 11, 13], "author": 1, "scope": 1, "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "represent": [1, 3], "document": [1, 3], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "job": 2, "metadata": [2, 3, 6, 9, 11], "submiss": 2, "tool": 2, "dr": [2, 9], "verifi": [2, 9], "ingest": [2, 9], "modul": [2, 10, 11], "gen3index": 3, "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "tupl": [3, 11, 12], "async": [3, 4, 5, 9, 11, 12], "async_create_record": 3, "hash": [3, 11], "size": [3, 10, 11], "did": 3, "baseid": 3, "acl": [3, 11], "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "creat": [3, 4, 5, 6, 8, 10, 11], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "md5": [3, 11], "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 10, 11], "entri": 3, "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "async_get_record": 3, "async_get_records_from_checksum": 3, "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "async_get_records_on_pag": 3, "async_get_with_param": 3, "param": [3, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "async_query_url": 3, "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "async_update_record": 3, "updat": [3, 5, 10, 11], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "create_blank": 3, "blank": 3, "create_new_vers": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "note": [3, 11, 12], "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "create_record": 3, "delete_record": [3, 8], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": 3, "distribut": 3, "identifi": [3, 9, 11], "boolean": 3, "we": [3, 11], "get_all_record": 3, "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "get_latest_vers": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "get_record": 3, "get_record_doc": 3, "repres": [3, 5, 10], "get_records_on_pag": 3, "get_stat": 3, "info": [3, 11], "get_url": 3, "algorithm": 3, "get_vers": [3, 4, 5], "get_with_param": 3, "file_st": 3, "client": 3, "like": [3, 5, 9, 11, 12], "signpost": 3, "is_healthi": [3, 4, 5], "healthi": [3, 4, 5], "query_url": 3, "update_blank": 3, "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "update_record": 3, "exist": [3, 5, 6, 9, 12], "gen3job": [4, 10], "s": [4, 8, 10, 11], "dispatch": 4, "async_run_job_and_wait": 4, "job_nam": 4, "job_input": 4, "kwarg": [4, 5], "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10], "global": 4, "input": [4, 10, 11], "create_job": 4, "get_output": 4, "job_id": 4, "previous": 4, "complet": [4, 11], "get_statu": 4, "statu": [4, 10], "bool": [4, 5, 8, 10, 11, 12], "list_job": 4, "gen3metadata": 5, "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "async_cr": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "async_get": 5, "async_upd": 5, "batch_creat": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "create_index_key_path": 5, "databas": 5, "delete_index_key_path": 5, "get_index_key_path": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": 5, "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": 5, "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "must": 5, "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "gen3object": 6, "level": 6, "current": [6, 8, 10], "allow": [6, 8, 10, 11, 12], "system": [6, 7, 8, 9], "delete_object": 6, "bucket": 6, "gen3queri": 7, "elasticsearch": 7, "graphql_queri": 7, "query_str": 7, "execut": [7, 8], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "http": [7, 11, 12], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "valid": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "raw_data_download": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "gen3submiss": 8, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "create_program": 8, "program": [8, 11], "create_project": 8, "project": [8, 11], "dcf": 8, "delete_nod": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "delete_program": 8, "empti": 8, "delete_project": 8, "uuid1": 8, "uuid2": 8, "export_nod": 8, "node_typ": 8, "fileformat": 8, "singl": [8, 10], "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "export_record": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "get_dictionary_al": 8, "entir": 8, "schema": 8, "get_dictionary_nod": 8, "get_graphql_schema": 8, "run": 8, "introspect": 8, "get_program": 8, "regist": 8, "get_project_dictionari": 8, "get_project_manifest": 8, "manifest": [8, 9, 10, 11, 12], "get_project": 8, "open_project": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "submit_fil": 8, "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "submit_record": 8, "should": [8, 11], "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": 9, "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "downloadmanag": 10, "datafil": 10, "load": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "i": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "hostnam": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "user_access": 10, "cache_hosts_wts_token": 10, "object_list": 10, "host": 10, "obtain": 10, "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "downloadstatu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "get_fresh_token": 10, "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolve_object": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "start_tim": 10, "end_tim": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "object_id": 10, "object_typ": 10, "drsobjecttyp": 10, "unknown": 10, "file_s": [10, 11], "updated_tim": 10, "created_tim": 10, "access_method": 10, "factori": 10, "children": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "_manag": 10, "self": 10, "pprint": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": 10, "md5sum": 10, "commons_url": [10, 11, 12], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "create_object_list": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "load_manifest": 10, "download_drs_object": 10, "output_dir": 10, "conveni": 10, "download_files_in_drs_manifest": 10, "infil": 10, "list_access_in_drs_manifest": 10, "list_drs_object": 10, "list_files_in_drs_manifest": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "separ": 11, "header": 11, "space": 11, "download_manifest": 11, "current_dir": 11, "indexd_record_page_s": 11, "per": [11, 12], "max_concurrent_request": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "tmp_folder": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "async_download_object_manifest": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "prev_guid": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "threadcontrol": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "delete_all_guid": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "index_object_manifest": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "loop": 11, "ha": 11, "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "altern": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "async_verify_object_manifest": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1654627682": [], "4302042": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "column_to_use_as_guid": 12, "guid_type_for_indexed_file_object": 12, "guid_type_for_non_indexed_file_object": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "async_ingest_metadata_manifest": 12, "metadata_sourc": 12, "672043": [], "get_guid_from_fil": 12, "metadata_typ": 12, "namespac": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "async_query_urls_from_indexd": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "gen3wsstorag": 13, "copi": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "download_url": 13, "ws": 13, "wskei": 13, "ls": 13, "ls_path": 13, "ws_urlstr": 13, "form": 13, "rm": 13, "rm_path": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "upload_url": 13, "1654634067": [], "5923028": [], "8967967": [], "1654789636": [], "6798043": [], "9773014": [], "1654807944": [11, 12], "3028057": 11, "560571": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_update"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "s": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 56}}) \ No newline at end of file +Search.setIndex({"docnames": ["auth", "file", "index", "indexing", "jobs", "metadata", "object", "query", "submission", "tools", "tools/drs_pull", "tools/indexing", "tools/metadata", "wss"], "filenames": ["auth.rst", "file.rst", "index.rst", "indexing.rst", "jobs.rst", "metadata.rst", "object.rst", "query.rst", "submission.rst", "tools.rst", "tools/drs_pull.rst", "tools/indexing.rst", "tools/metadata.rst", "wss.rst"], "titles": ["Gen3 Auth Helper", "Gen3 File Class", "Welcome to Gen3 SDK\u2019s documentation!", "Gen3 Index Class", "Gen3 Jobs Class", "Gen3 Metadata Class", "Gen3 Object Class", "Gen3 Query Class", "Gen3 Submission Class", "Gen3 Tools", "DRS Download Tools", "Indexing Tools", "Metadata Tools", "Gen3 Workspace Storage"], "terms": {"class": [0, 2, 10, 11, 13], "gen3auth": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "endpoint": [0, 1, 3, 4, 5, 7, 8, 13], "none": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "refresh_fil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "refresh_token": 0, "idp": 0, "sourc": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "base": [0, 1, 3, 4, 5, 6, 7, 8, 9, 11, 13], "authbas": 0, "us": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "request": [0, 1, 3, 8, 11, 12], "implement": 0, "order": [0, 8], "support": [0, 1, 5, 8, 11], "jwt": 0, "authent": 0, "gener": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 13], "access": [0, 1, 3, 7, 10], "token": [0, 10], "from": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "provid": [0, 1, 3, 5, 7, 8, 12], "refresh": [0, 10], "file": [0, 2, 3, 4, 8, 9, 10, 11, 12, 13], "string": [0, 3, 5, 11, 13], "automat": 0, "when": [0, 3, 7, 10, 12], "thei": [0, 10], "expir": [0, 1], "paramet": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "str": [0, 1, 3, 4, 5, 7, 8, 10, 11, 12], "opt": 0, "The": [0, 1, 2, 3, 5, 8, 10, 11], "contain": [0, 2, 5, 8, 9, 10, 11, 12], "download": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "json": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "web": 0, "option": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "work": [0, 10], "workspac": [0, 2], "default": [0, 1, 3, 7, 8, 11, 12], "env": 0, "gen3_api_kei": 0, "credenti": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "set": [0, 1, 10], "includ": [0, 3], "search": [0, 2, 3], "path": [0, 1, 5, 10, 11, 13], "valu": [0, 1, 3, 5, 7, 10, 11], "doe": [0, 12], "interpret": 0, "wt": [0, 10], "an": [0, 3, 6, 8, 10], "accesstoken": 0, "If": [0, 1, 7, 11, 12], "can": [0, 3, 4, 8, 11, 12], "specifi": [0, 3, 11, 13], "local": [0, 13], "indic": [0, 11], "environ": 0, "fenc": [0, 1], "exampl": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "point": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "sandbox": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "common": [0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], "while": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "profil": [0, 1, 3, 4, 5, 6, 7, 8, 10, 13], "page": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 13], "instal": [0, 2], "crdc": 0, "some": [0, 2], "arbitrari": 0, "kei": [0, 3, 5, 13], "variabl": [0, 7, 8], "rather": 0, "than": [0, 5], "pass": [0, 7, 8, 10], "argument": [0, 13], "constructor": 0, "initi": [0, 10], "follow": [0, 11], "curl": 0, "data": [0, 1, 3, 5, 7, 8, 10, 11], "given": [0, 3, 4, 5, 8, 10, 12, 13], "ex": [0, 11, 12], "user": [0, 10, 12], "return": [0, 1, 3, 4, 5, 6, 7, 8, 10, 11], "respons": [0, 1, 3, 4], "under": [0, 8, 13], "index": [0, 2, 5, 9], "authz": [0, 1, 3, 10, 11], "map": [0, 11], "get": [0, 1, 3, 4, 5, 8, 10, 11, 12, 13], "post": [0, 11], "put": 0, "delet": [0, 1, 3, 5, 6, 8, 10, 11], "els": [0, 12], "filenam": [0, 8, 10, 11, 12], "get_access_token": 0, "auto": [0, 2], "within": [0, 2, 9], "5": [0, 5], "minut": 0, "refresh_access_token": 0, "new": [0, 3], "gen3fil": 1, "auth_provid": [1, 3, 4, 5, 6, 7, 8, 13], "object": [1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 13], "For": [1, 5, 6, 7, 8, 9, 11], "interact": [1, 3, 4, 5, 6, 8, 13], "manag": [1, 5, 10], "featur": [1, 6], "A": [1, 3, 4, 5, 6, 7, 8, 10, 11, 13], "servic": [1, 3, 4, 5, 6, 8, 11, 12, 13], "presign": 1, "url": [1, 3, 9, 10, 11, 12, 13], "right": 1, "now": [1, 8], "instanc": [1, 3, 6, 7, 8, 9, 10], "auth": [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13], "delete_fil": 1, "guid": [1, 3, 5, 6, 11, 12], "method": [1, 7, 10], "deprec": 1, "delete_file_loc": [1, 6], "instead": [1, 7, 11], "all": [1, 3, 4, 5, 6, 7, 8, 10, 11, 12], "locat": [1, 6], "store": [1, 3, 10], "remov": [1, 6, 11, 13], "its": [1, 3], "record": [1, 3, 5, 7, 8, 11, 12], "indexd": [1, 3, 6, 10, 11, 12], "uuid": [1, 3, 8], "id": [1, 3, 10, 11], "text": [1, 7, 8], "result": [1, 8, 10], "type": [1, 3, 4, 5, 7, 8, 10, 11, 12], "get_presigned_url": 1, "protocol": 1, "retriev": [1, 8, 10, 12], "give": 1, "limit": [1, 3, 5, 12], "time": [1, 8, 10, 11], "pick": 1, "avail": [1, 2, 10, 11], "queri": [1, 2, 3, 5, 8, 11, 12], "upload_fil": 1, "file_nam": [1, 3, 10, 11], "expires_in": 1, "upload": [1, 3, 8, 13], "list": [1, 3, 4, 5, 7, 8, 10, 11, 13], "author": 1, "scope": 1, "storag": [1, 2, 6], "s3": [1, 10, 11], "az": 1, "isn": 1, "t": [1, 5, 11], "int": [1, 3, 5, 7, 8, 10, 11, 12], "amount": [1, 9], "second": [1, 4], "sign": 1, "datetim": [1, 10], "utcnow": 1, "Be": 1, "sure": 1, "posit": [1, 7], "integ": [1, 3, 8], "also": 1, "treat": [1, 5], "max_presigned_url_ttl": 1, "configur": 1, "represent": [1, 3], "document": [1, 3], "readm": 2, "python": [2, 9, 11], "github": [2, 7], "repo": 2, "setup": 2, "inform": [2, 3, 10], "along": 2, "quickstart": 2, "script": 2, "more": [2, 5, 7, 9, 10], "detail": [2, 7, 10], "about": [2, 3], "variou": 2, "function": [2, 3, 4, 5, 9, 10, 11, 12], "mostli": 2, "docstr": 2, "code": [2, 8], "helper": 2, "job": 2, "metadata": [2, 3, 6, 9, 11], "submiss": 2, "tool": 2, "dr": [2, 9], "verifi": [2, 9], "ingest": [2, 9], "modul": [2, 10, 11], "gen3index": 3, "service_loc": [3, 4, 5], "public": [3, 5], "read": [3, 5, 11], "onli": [3, 5, 7, 8, 10, 11], "necessari": [3, 5], "basic": [3, 11, 12], "cred": 3, "tupl": [3, 11, 12], "async": [3, 4, 5, 9, 11, 12], "async_create_record": 3, "hash": [3, 11], "size": [3, 10, 11], "did": 3, "baseid": 3, "acl": [3, 11], "urls_metadata": 3, "version": [3, 4, 5], "_ssl": [3, 4, 5], "asynchron": [3, 4, 5], "creat": [3, 4, 5, 6, 8, 10, 11], "dict": [3, 4, 5, 10, 11, 12], "eg": 3, "md5": [3, 11], "ab167e49d25b488939b1ede42752458b": 3, "associ": [3, 5], "made": 3, "where": [3, 5, 11, 12], "you": [3, 8, 11], "control": 3, "rbac": 3, "name": [3, 4, 8, 10, 11, 12, 13], "addit": [3, 10, 11], "entri": 3, "attach": [3, 5], "each": [3, 8, 10, 11], "group": 3, "previou": [3, 11], "async_get_record": 3, "async_get_records_from_checksum": 3, "checksum": [3, 10], "checksum_typ": 3, "match": [3, 5, 12], "async_get_records_on_pag": 3, "async_get_with_param": 3, "param": [3, 8, 10], "correspond": 3, "suppli": 3, "need": [3, 7, 10, 11], "handl": [3, 10], "hash_typ": 3, "whether": [3, 4, 5, 8, 11, 12], "ssl": [3, 4, 5], "async_query_url": 3, "pattern": [3, 12], "against": [3, 7, 8, 11, 12], "async_update_record": 3, "updat": [3, 5, 10, 11], "bodi": 3, "dictionari": [3, 4, 5, 7, 8], "format": [3, 5, 8, 11], "create_blank": 3, "blank": 3, "create_new_vers": 3, "add": [3, 5], "sinc": 3, "content": [3, 13], "immut": 3, "want": [3, 8], "chang": [3, 11], "That": 3, "field": [3, 5, 7, 11, 12], "old": 3, "ad": 3, "least": 3, "one": [3, 5, 10, 11], "suffici": 3, "note": [3, 11, 12], "good": 3, "idea": 3, "number": [3, 7, 8, 11, 12], "create_record": 3, "delete_record": [3, 8], "noth": [3, 6], "dist_resolut": 3, "true": [3, 4, 5, 6, 7, 8, 10, 11, 12], "alia": 3, "distribut": 3, "identifi": [3, 9, 11], "boolean": 3, "we": [3, 11], "get_all_record": 3, "pagin": 3, "fals": [3, 5, 6, 10, 11], "start": [3, 4, 7, 8, 10], "get_latest_vers": 3, "has_vers": 3, "latest": 3, "exclud": 3, "without": [3, 5], "get_record": 3, "get_record_doc": 3, "repres": [3, 5, 10], "get_records_on_pag": 3, "get_stat": 3, "info": [3, 11], "get_url": 3, "algorithm": 3, "get_vers": [3, 4, 5], "get_with_param": 3, "file_st": 3, "client": 3, "like": [3, 5, 9, 11, 12], "signpost": 3, "is_healthi": [3, 4, 5], "healthi": [3, 4, 5], "query_url": 3, "update_blank": 3, "rev": 3, "revis": 3, "simpl": 3, "consist": 3, "mechan": 3, "update_record": 3, "exist": [3, 5, 6, 9, 12], "gen3job": [4, 10], "s": [4, 8, 10, 11], "dispatch": 4, "async_run_job_and_wait": 4, "job_nam": 4, "job_input": 4, "kwarg": [4, 5], "wait": 4, "output": [4, 5, 11, 12], "Will": [4, 7, 10], "sleep": 4, "linear": 4, "delai": 4, "until": [4, 10], "done": 4, "1": [4, 5, 8, 10], "global": 4, "input": [4, 10, 11], "create_job": 4, "get_output": 4, "job_id": 4, "previous": 4, "complet": [4, 11], "get_statu": 4, "statu": [4, 10], "bool": [4, 5, 8, 10, 11, 12], "list_job": 4, "gen3metadata": 5, "md": [5, 7, 10, 12], "admin_endpoint_suffix": 5, "admin": [5, 11], "async_cr": 5, "overwrit": 5, "what": 5, "end": [5, 10], "up": [5, 9], "blob": [5, 7], "async_get": 5, "async_upd": 5, "batch_creat": 5, "metadata_list": 5, "specif": [5, 8, 11, 12], "expect": [5, 9, 11], "anoth": 5, "create_index_key_path": 5, "databas": 5, "delete_index_key_path": 5, "get_index_key_path": 5, "return_full_metadata": 5, "10": [5, 7, 11], "offset": [5, 7], "0": [5, 7, 8, 11], "use_agg_md": 5, "off": 5, "logic": [5, 12], "filter": [5, 7], "b": [5, 11], "2": 5, "have": [5, 11], "ar": [5, 7, 8, 9, 10, 11], "alwai": 5, "nest": 5, "c": [5, 11], "3": 5, "same": [5, 11, 13], "whose": 5, "ani": [5, 10, 11], "But": 5, "differ": 5, "must": 5, "33": 5, "d": 5, "4": [5, 11], "e": [5, 10], "won": 5, "333": 5, "defin": [5, 8, 10], "api": [5, 8, 11], "just": [5, 11, 12], "max": 5, "num": 5, "OR": 5, "gen3object": 6, "level": 6, "current": [6, 8, 10], "allow": [6, 8, 10, 11, 12], "system": [6, 7, 8, 9], "delete_object": 6, "bucket": 6, "gen3queri": 7, "elasticsearch": 7, "graphql_queri": 7, "query_str": 7, "execut": [7, 8], "graphql": [7, 8], "query_txt": [7, 8], "syntax": 7, "see": [7, 10, 11], "http": [7, 11, 12], "com": 7, "uc": 7, "cdi": 7, "guppi": 7, "master": 7, "doc": [7, 10], "data_typ": 7, "my_index": 7, "my_field": 7, "first": [7, 8], "filter_object": 7, "sort_object": 7, "verbos": [7, 8], "row": [7, 8, 11, 12], "sort": 7, "equal": 7, "respect": 7, "complex": 7, "appli": 7, "One": 7, "unaccess": 7, "valid": 7, "regular": 7, "tier": 7, "mode": 7, "subject": [7, 8], "50": 7, "vital_statu": 7, "submitter_id": 7, "aliv": 7, "asc": 7, "raw_data_download": 7, "sort_field": 7, "raw": [7, 11], "project_id": [7, 8], "my_program": 7, "my_project": 7, "gen3submiss": 8, "submit": [8, 11], "export": [8, 10], "sheepdog": 8, "through": [8, 11], "peregrin": 8, "sub": 8, "create_program": 8, "program": [8, 11], "create_project": 8, "project": [8, 11], "dcf": 8, "delete_nod": 8, "node_nam": 8, "batch_siz": 8, "100": [8, 11], "node": 8, "how": [8, 11], "mani": [8, 11], "print": [8, 10], "progress": [8, 10], "log": [8, 10, 11, 12], "ccle": 8, "demograph": 8, "ordered_node_list": 8, "revers": 8, "graph": 8, "experi": 8, "delete_program": 8, "empti": 8, "delete_project": 8, "uuid1": 8, "uuid2": 8, "export_nod": 8, "node_typ": 8, "fileformat": 8, "singl": [8, 10], "which": [8, 10], "belong": 8, "either": 8, "tsv": [8, 11, 12], "screen": 8, "sampl": [8, 10], "ccle_sample_nod": 8, "export_record": 8, "d70b41b9": 8, "6f90": 8, "4714": 8, "8420": 8, "e043ab8b77b9": 8, "ccle_one_record": 8, "get_dictionary_al": 8, "entir": 8, "schema": 8, "get_dictionary_nod": 8, "get_graphql_schema": 8, "run": 8, "introspect": 8, "get_program": 8, "regist": 8, "get_project_dictionari": 8, "get_project_manifest": 8, "manifest": [8, 9, 10, 11, 12], "get_project": 8, "open_project": 8, "mark": 8, "open": [8, 10, 11], "mean": 8, "etc": 8, "max_tri": 8, "retri": 8, "fail": [8, 10], "submit_fil": 8, "chunk_siz": 8, "30": 8, "row_offset": 8, "spreadsheet": 8, "multipl": [8, 11], "csv": [8, 11, 12], "xlsx": 8, "worksheet": 8, "skip": 8, "data_spreadsheet": 8, "submit_record": 8, "should": [8, 11], "arrai": 8, "action": [9, 11], "These": 9, "broken": 9, "broad": 9, "categori": 9, "task": 9, "relat": 9, "persist": 9, "Such": 9, "mai": 9, "involv": 9, "assign": 9, "everi": [9, 11], "alreadi": 9, "larg": 9, "most": 9, "util": 9, "capabl": 9, "make": [9, 11], "effici": 9, "main": 10, "downloadmanag": 10, "datafil": 10, "load": 10, "manifest_1": 10, "my_common": 10, "org": 10, "gen3": [10, 11, 12], "my_credenti": 10, "i": 10, "howto": 10, "drsdownload": 10, "drs_download": 10, "hostnam": 10, "download_list": 10, "show_progress": 10, "assist": 10, "minimum": 10, "interest": 10, "user_access": 10, "cache_hosts_wts_token": 10, "object_list": 10, "host": 10, "obtain": 10, "It": 10, "possibl": 10, "save_directori": 10, "unpack_packag": 10, "delete_unpacked_packag": 10, "directori": [10, 11], "call": [10, 13], "case": 10, "error": [10, 11, 12], "continu": 10, "downloadstatu": 10, "save": 10, "show": 10, "bar": 10, "disabl": 10, "unpack": 10, "packag": 10, "after": 10, "them": [10, 11], "get_fresh_token": 10, "drs_hostnam": 10, "known": 10, "otherwis": 10, "success": 10, "resolve_object": 10, "resolv": 10, "permiss": 10, "wai": 10, "determin": [10, 11, 12], "reason": 10, "pend": 10, "start_tim": 10, "end_tim": 10, "objectid": 10, "datamanag": 10, "occur": 10, "attribut": [10, 11], "object_id": 10, "object_typ": 10, "drsobjecttyp": 10, "unknown": 10, "file_s": [10, 11], "updated_tim": 10, "created_tim": 10, "access_method": 10, "factori": 10, "children": 10, "popul": [10, 12], "server": 10, "requir": 10, "byte": 10, "timestamp": 10, "last": 10, "g": 10, "child": 10, "bundl": 10, "_manag": 10, "self": 10, "pprint": 10, "indent": 10, "pretti": 10, "In": 10, "similar": 10, "linux": 10, "tree": 10, "command": 10, "md5sum": 10, "commons_url": [10, 11, 12], "typic": 10, "discoveri": 10, "cach": 10, "files": 10, "static": 10, "create_object_list": 10, "bownload": 10, "successfulli": 10, "pars": [10, 11, 12, 13], "load_manifest": 10, "download_drs_object": 10, "output_dir": 10, "conveni": 10, "download_files_in_drs_manifest": 10, "infil": 10, "list_access_in_drs_manifest": 10, "list_drs_object": 10, "list_files_in_drs_manifest": 10, "wrapper": 10, "cli": 10, "process": 11, "coroutin": 11, "asyncio": [11, 12], "librari": 11, "comma": 11, "separ": 11, "header": 11, "space": 11, "download_manifest": 11, "current_dir": 11, "indexd_record_page_s": 11, "per": [11, 12], "max_concurrent_request": [11, 12], "maximum": [11, 12], "desir": 11, "concurr": [11, 12], "across": 11, "thread": 11, "tmp_folder": 11, "folder": 11, "place": 11, "temporari": 11, "write": 11, "safe": 11, "so": 11, "To": 11, "workaround": 11, "concat": 11, "async_download_object_manifest": 11, "output_filenam": [11, 12], "num_process": 11, "24": [11, 12], "input_manifest": 11, "root": [11, 12], "domain": [11, 12], "live": [11, 12], "parallel": 11, "hit": 11, "total": 11, "help": 11, "everyth": 11, "attempt": 11, "tab": 11, "session": 11, "column": [11, 12], "shown": 11, "below": 11, "255e396f": 11, "f1f8": 11, "11e9": 11, "9a07": 11, "0a80fada099c": 11, "473d83400bc1bc9dc635e334faddf33c": 11, "363455714": 11, "pdcdatastor": 11, "test1": 11, "0a80fada098c": 11, "473d83400bc1bc9dc635e334faddd33c": 11, "343434344": 11, "test2": 11, "0a80fada097c": 11, "473d83400bc1bc9dc635e334fadd433c": 11, "543434443": 11, "phs0001": 11, "phs0002": 11, "test3": 11, "0a80fada096c": 11, "test4": 11, "0a80fada010c": 11, "473d83400bc1bc9dc635e334fadde33c": 11, "test5": 11, "index_manifest": 11, "prev_guid": 11, "usag": 11, "py": 11, "giangb": 11, "planx": 11, "pla": 11, "net": 11, "manifest_fil": [11, 12], "path_to_manifest": 11, "replace_url": 11, "thread_num": 11, "api_kei": 11, "threadcontrol": 11, "processed_fil": 11, "num_total_fil": 11, "synchron": 11, "delete_all_guid": 11, "warn": 11, "IF": 11, "know": 11, "THE": 11, "implic": 11, "index_object_manifest": 11, "manifest_file_delimit": [11, 12], "submit_additional_metadata_column": 11, "loop": 11, "ha": 11, "password": [11, 12], "flag": 11, "replac": 11, "delimit": 11, "guid_exampl": 11, "md5_hash": 11, "fieldnam": 11, "There": 11, "those": 11, "abov": 11, "overrid": [11, 12], "assum": 11, "altern": 11, "behavior": 11, "simpli": 11, "manifest_row_pars": [11, 12], "custom": 11, "import": 11, "verify_manifest": 11, "def": 11, "_get_authz_from_row": 11, "strip": 11, "parser": [11, 12], "verify_object_manifest": 11, "verif": 11, "error_nam": 11, "value_from_manifest": 11, "actual": 11, "value_from_indexd": 11, "93d9af72": 11, "b0f1": 11, "450c": 11, "a5c6": 11, "7d3d8d2083b4": 11, "dev": 11, "test": 11, "async_verify_object_manifest": 11, "_get_acl_from_row": 11, "_get_file_name_from_row": 11, "_get_file_size_from_row": 11, "_get_guid_from_row": 11, "_get_md5_from_row": 11, "_get_urls_from_row": 11, "1654627682": [], "4302042": [], "indexd_field": [11, 12], "func_to_parse_row": [11, 12], "delimet": [11, 12], "metdata": 12, "ingest_manifest": 12, "column_to_use_as_guid": 12, "guid_type_for_indexed_file_object": 12, "guid_type_for_non_indexed_file_object": 12, "NOT": 12, "guid_from_fil": 12, "_get_guid_for_row": 12, "indexed_file_object_guid": 12, "_query_for_associated_indexd_record_guid": 12, "guid_for_row": 12, "elsewher": 12, "async_ingest_metadata_manifest": 12, "metadata_sourc": 12, "672043": [], "get_guid_from_fil": 12, "metadata_typ": 12, "namespac": 12, "dbgap": 12, "usual": 12, "fill": 12, "_guid_typ": 12, "is_indexed_file_object": 12, "async_query_urls_from_indexd": 12, "lock": 12, "semaphor": 12, "semaphon": 12, "ammount": 12, "connect": 12, "wss": 13, "gen3wsstorag": 13, "copi": 13, "src_urlstr": 13, "dest_urlstr": 13, "appropri": 13, "src_w": 13, "src_wskei": 13, "dest_path": 13, "disk": 13, "download_url": 13, "ws": 13, "wskei": 13, "ls": 13, "ls_path": 13, "ws_urlstr": 13, "form": 13, "rm": 13, "rm_path": 13, "src_path": 13, "dest_w": 13, "dest_wskei": 13, "upload_url": 13, "1654634067": [], "5923028": [], "8967967": [], "1654789636": [], "6798043": [], "9773014": [], "1654807944": [], "3028057": [], "560571": [], "1654874610": [], "345438": [], "6664228": [], "1654874771": [11, 12], "190345": 11, "4957037": 12}, "objects": {"gen3.auth": [[0, 0, 1, "", "Gen3Auth"]], "gen3.auth.Gen3Auth": [[0, 1, 1, "", "curl"], [0, 1, 1, "", "get_access_token"], [0, 1, 1, "", "refresh_access_token"]], "gen3.file": [[1, 0, 1, "", "Gen3File"]], "gen3.file.Gen3File": [[1, 1, 1, "", "delete_file"], [1, 1, 1, "", "delete_file_locations"], [1, 1, 1, "", "get_presigned_url"], [1, 1, 1, "", "upload_file"]], "gen3.index": [[3, 0, 1, "", "Gen3Index"]], "gen3.index.Gen3Index": [[3, 1, 1, "", "async_create_record"], [3, 1, 1, "", "async_get_record"], [3, 1, 1, "", "async_get_records_from_checksum"], [3, 1, 1, "", "async_get_records_on_page"], [3, 1, 1, "", "async_get_with_params"], [3, 1, 1, "", "async_query_urls"], [3, 1, 1, "", "async_update_record"], [3, 1, 1, "", "create_blank"], [3, 1, 1, "", "create_new_version"], [3, 1, 1, "", "create_record"], [3, 1, 1, "", "delete_record"], [3, 1, 1, "", "get"], [3, 1, 1, "", "get_all_records"], [3, 1, 1, "", "get_latest_version"], [3, 1, 1, "", "get_record"], [3, 1, 1, "", "get_record_doc"], [3, 1, 1, "", "get_records"], [3, 1, 1, "", "get_records_on_page"], [3, 1, 1, "", "get_stats"], [3, 1, 1, "", "get_urls"], [3, 1, 1, "", "get_version"], [3, 1, 1, "", "get_versions"], [3, 1, 1, "", "get_with_params"], [3, 1, 1, "", "is_healthy"], [3, 1, 1, "", "query_urls"], [3, 1, 1, "", "update_blank"], [3, 1, 1, "", "update_record"]], "gen3.jobs": [[4, 0, 1, "", "Gen3Jobs"]], "gen3.jobs.Gen3Jobs": [[4, 1, 1, "", "async_run_job_and_wait"], [4, 1, 1, "", "create_job"], [4, 1, 1, "", "get_output"], [4, 1, 1, "", "get_status"], [4, 1, 1, "", "get_version"], [4, 1, 1, "", "is_healthy"], [4, 1, 1, "", "list_jobs"]], "gen3.metadata": [[5, 0, 1, "", "Gen3Metadata"]], "gen3.metadata.Gen3Metadata": [[5, 1, 1, "", "async_create"], [5, 1, 1, "", "async_get"], [5, 1, 1, "", "async_update"], [5, 2, 1, "", "auth_provider"], [5, 1, 1, "", "batch_create"], [5, 1, 1, "", "create"], [5, 1, 1, "", "create_index_key_path"], [5, 1, 1, "", "delete"], [5, 1, 1, "", "delete_index_key_path"], [5, 2, 1, "", "endpoint"], [5, 1, 1, "", "get"], [5, 1, 1, "", "get_index_key_paths"], [5, 1, 1, "", "get_version"], [5, 1, 1, "", "is_healthy"], [5, 1, 1, "", "query"], [5, 1, 1, "", "update"]], "gen3.object": [[6, 0, 1, "", "Gen3Object"]], "gen3.object.Gen3Object": [[6, 1, 1, "", "delete_object"]], "gen3.query": [[7, 0, 1, "", "Gen3Query"]], "gen3.query.Gen3Query": [[7, 1, 1, "", "graphql_query"], [7, 1, 1, "", "query"], [7, 1, 1, "", "raw_data_download"]], "gen3.submission": [[8, 0, 1, "", "Gen3Submission"]], "gen3.submission.Gen3Submission": [[8, 1, 1, "", "create_program"], [8, 1, 1, "", "create_project"], [8, 1, 1, "", "delete_node"], [8, 1, 1, "", "delete_nodes"], [8, 1, 1, "", "delete_program"], [8, 1, 1, "", "delete_project"], [8, 1, 1, "", "delete_record"], [8, 1, 1, "", "delete_records"], [8, 1, 1, "", "export_node"], [8, 1, 1, "", "export_record"], [8, 1, 1, "", "get_dictionary_all"], [8, 1, 1, "", "get_dictionary_node"], [8, 1, 1, "", "get_graphql_schema"], [8, 1, 1, "", "get_programs"], [8, 1, 1, "", "get_project_dictionary"], [8, 1, 1, "", "get_project_manifest"], [8, 1, 1, "", "get_projects"], [8, 1, 1, "", "open_project"], [8, 1, 1, "", "query"], [8, 1, 1, "", "submit_file"], [8, 1, 1, "", "submit_record"]], "gen3": [[9, 3, 0, "-", "tools"]], "gen3.tools.download": [[10, 3, 0, "-", "drs_download"]], "gen3.tools.download.drs_download": [[10, 0, 1, "", "DownloadManager"], [10, 0, 1, "", "DownloadStatus"], [10, 0, 1, "", "Downloadable"], [10, 0, 1, "", "Manifest"], [10, 4, 1, "", "download_drs_object"], [10, 4, 1, "", "download_files_in_drs_manifest"], [10, 4, 1, "", "list_access_in_drs_manifest"], [10, 4, 1, "", "list_drs_object"], [10, 4, 1, "", "list_files_in_drs_manifest"]], "gen3.tools.download.drs_download.DownloadManager": [[10, 1, 1, "", "cache_hosts_wts_tokens"], [10, 1, 1, "", "download"], [10, 1, 1, "", "get_fresh_token"], [10, 1, 1, "", "resolve_objects"], [10, 1, 1, "", "user_access"]], "gen3.tools.download.drs_download.DownloadStatus": [[10, 2, 1, "", "end_time"], [10, 2, 1, "", "start_time"], [10, 2, 1, "", "status"]], "gen3.tools.download.drs_download.Downloadable": [[10, 2, 1, "", "_manager"], [10, 2, 1, "", "access_methods"], [10, 2, 1, "", "children"], [10, 2, 1, "", "created_time"], [10, 1, 1, "", "download"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 2, 1, "", "hostname"], [10, 2, 1, "", "object_id"], [10, 2, 1, "", "object_type"], [10, 1, 1, "", "pprint"], [10, 2, 1, "", "updated_time"]], "gen3.tools.download.drs_download.Manifest": [[10, 2, 1, "", "commons_url"], [10, 1, 1, "", "create_object_list"], [10, 2, 1, "", "file_name"], [10, 2, 1, "", "file_size"], [10, 1, 1, "", "load"], [10, 1, 1, "", "load_manifest"], [10, 2, 1, "", "md5sum"], [10, 2, 1, "", "object_id"]], "gen3.tools.indexing": [[11, 3, 0, "-", "download_manifest"], [11, 3, 0, "-", "index_manifest"], [11, 3, 0, "-", "verify_manifest"]], "gen3.tools.indexing.download_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "INDEXD_RECORD_PAGE_SIZE"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 2, 1, "", "TMP_FOLDER"], [11, 4, 1, "", "async_download_object_manifest"]], "gen3.tools.indexing.index_manifest": [[11, 2, 1, "", "ACLS"], [11, 2, 1, "", "AUTHZ"], [11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "GUID"], [11, 2, 1, "", "MD5"], [11, 2, 1, "", "PREV_GUID"], [11, 2, 1, "", "SIZE"], [11, 0, 1, "", "ThreadControl"], [11, 2, 1, "", "URLS"], [11, 4, 1, "", "delete_all_guids"], [11, 4, 1, "", "index_object_manifest"]], "gen3.tools.indexing.verify_manifest": [[11, 2, 1, "", "CURRENT_DIR"], [11, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [11, 4, 1, "", "async_verify_object_manifest"]], "gen3.tools.metadata": [[12, 3, 0, "-", "ingest_manifest"]], "gen3.tools.metadata.ingest_manifest": [[12, 2, 1, "", "COLUMN_TO_USE_AS_GUID"], [12, 2, 1, "", "GUID_TYPE_FOR_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"], [12, 2, 1, "", "MAX_CONCURRENT_REQUESTS"], [12, 4, 1, "", "async_ingest_metadata_manifest"], [12, 4, 1, "", "async_query_urls_from_indexd"]], "gen3.wss": [[13, 0, 1, "", "Gen3WsStorage"]], "gen3.wss.Gen3WsStorage": [[13, 1, 1, "", "copy"], [13, 1, 1, "", "download"], [13, 1, 1, "", "download_url"], [13, 1, 1, "", "ls"], [13, 1, 1, "", "ls_path"], [13, 1, 1, "", "rm"], [13, 1, 1, "", "rm_path"], [13, 1, 1, "", "upload"], [13, 1, 1, "", "upload_url"]]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:attribute", "3": "py:module", "4": "py:function"}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "attribute", "Python attribute"], "3": ["py", "module", "Python module"], "4": ["py", "function", "Python function"]}, "titleterms": {"gen3": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 13], "auth": 0, "helper": 0, "file": 1, "class": [1, 3, 4, 5, 6, 7, 8], "welcom": 2, "sdk": 2, "s": 2, "document": 2, "indic": 2, "tabl": 2, "index": [3, 11], "job": 4, "metadata": [5, 12], "object": 6, "queri": 7, "submiss": 8, "tool": [9, 10, 11, 12], "dr": 10, "download": [10, 11], "verifi": 11, "ingest": 12, "workspac": 13, "storag": 13}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1, "sphinx": 56}}) \ No newline at end of file diff --git a/docs/_build/html/tools/indexing.html b/docs/_build/html/tools/indexing.html index 16360495c..f88ec8d60 100644 --- a/docs/_build/html/tools/indexing.html +++ b/docs/_build/html/tools/indexing.html @@ -340,7 +340,7 @@

Indexing Tools
-async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1654807944.3028057.log')[source]
+async gen3.tools.indexing.verify_manifest.async_verify_object_manifest(commons_url, manifest_file, max_concurrent_requests=24, manifest_row_parsers={'acl': <function _get_acl_from_row>, 'authz': <function _get_authz_from_row>, 'file_name': <function _get_file_name_from_row>, 'file_size': <function _get_file_size_from_row>, 'guid': <function _get_guid_from_row>, 'md5': <function _get_md5_from_row>, 'urls': <function _get_urls_from_row>}, manifest_file_delimiter=None, output_filename='verify-manifest-errors-1654874771.190345.log')[source]

Verify all file object records into a manifest csv

Parameters:
diff --git a/docs/_build/html/tools/metadata.html b/docs/_build/html/tools/metadata.html index da4286ad3..489f2422c 100644 --- a/docs/_build/html/tools/metadata.html +++ b/docs/_build/html/tools/metadata.html @@ -104,7 +104,7 @@

Metadata Tools
-async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1654807944.560571.log', get_guid_from_file=True, metadata_type=None)[source]
+async gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest(commons_url, manifest_file, metadata_source, auth=None, max_concurrent_requests=24, manifest_row_parsers={'guid_for_row': <function _get_guid_for_row>, 'indexed_file_object_guid': <function _query_for_associated_indexd_record_guid>}, manifest_file_delimiter=None, output_filename='ingest-metadata-manifest-errors-1654874771.4957037.log', get_guid_from_file=True, metadata_type=None)[source]

Ingest all metadata records into a manifest csv

Parameters:
diff --git a/docs/howto/diirmIndexing.md b/docs/howto/diirmIndexing.md index 16ce39332..400f8f0b1 100644 --- a/docs/howto/diirmIndexing.md +++ b/docs/howto/diirmIndexing.md @@ -18,6 +18,7 @@ import asyncio from gen3.tools import indexing from gen3.tools.indexing.verify_manifest import manifest_row_parsers +from gen3.utils import get_or_create_event_loop_for_thread logging.basicConfig(filename="output.log", level=logging.DEBUG) logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) @@ -25,8 +26,8 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) COMMONS = "https://{{insert-commons-here}}/" def main(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() + loop.run_until_complete( indexing.async_download_object_manifest( @@ -83,8 +84,8 @@ def main(): # override default parsers manifest_row_parsers["file_size"] = _get_file_size - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() + loop.run_until_complete( indexing.async_verify_object_manifest( diff --git a/docs/howto/metadataTools.md b/docs/howto/metadataTools.md index e839a5b93..5a41f41b9 100644 --- a/docs/howto/metadataTools.md +++ b/docs/howto/metadataTools.md @@ -30,6 +30,7 @@ import asyncio from gen3.auth import Gen3Auth from gen3.tools import metadata from gen3.tools.metadata.ingest_manifest import manifest_row_parsers +from gen3.utils import get_or_create_event_loop_for_thread logging.basicConfig(filename="output.log", level=logging.DEBUG) logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) @@ -39,8 +40,8 @@ logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) MANIFEST = "dbgap_extract_guid.tsv" def main(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() + auth = Gen3Auth(refresh_file="credentials.json") @@ -256,8 +257,8 @@ MANIFEST = "dbgap_extract.tsv" def main(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() + auth = Gen3Auth(refresh_file="credentials.json") @@ -565,8 +566,8 @@ def get_dbgap_merged_metadata_manifest(): "partial_match_or_exact_match": "partial_match", } - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() + job_output = loop.run_until_complete( jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=job_input) @@ -591,8 +592,8 @@ def metadata_ingest(): "metadata_source": "dbgap", } - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() + job_output = loop.run_until_complete( jobs.async_run_job_and_wait(job_name=INGEST_METADATA_JOB, job_input=job_input) @@ -633,8 +634,8 @@ COMMONS = "https://{{insert-commons-here}}/" def main(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() + loop.run_until_complete( metadata.async_verify_metadata_manifest( diff --git a/docs/reference/sdkClasses.md b/docs/reference/sdkClasses.md index e5f270378..713cfe22c 100644 --- a/docs/reference/sdkClasses.md +++ b/docs/reference/sdkClasses.md @@ -17,7 +17,7 @@ This contains an auth wrapper for supporting JWT based authentication with `requ By default - the `Gen3Auth` constructor looks for an api key in `~/.gen3/credentials.json`. You may override that path via the `GEN3_API_KEY` environment varialbe, or by passing a -`refresh_file` parameter. +`refresh_file` parameter. When working in a Gen3 Workspace, all parameters are optional and the `Gen3Auth` instance should be initialized as follows: @@ -52,6 +52,7 @@ import asyncio from gen3.index import Gen3Index from gen3.auth import Gen3Auth from gen3.jobs import Gen3Jobs, DBGAP_METADATA_JOB, INGEST_METADATA_JOB +from gen3.utils import get_or_create_event_loop_for_thread # An API Key downloaded from the above commons' "Profile" page API_KEY_FILEPATH = "credentials.json" @@ -69,8 +70,7 @@ def metadata_ingest(): "metadata_source": "dbgaptest", } - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() job_output = loop.run_until_complete( jobs.async_run_job_and_wait(job_name=INGEST_METADATA_JOB, job_input=job_input) @@ -93,8 +93,7 @@ def main(): "partial_match_or_exact_match": "partial_match", } - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() job_output = loop.run_until_complete( jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=job_input) @@ -138,8 +137,7 @@ def example_async_run_job(): auth = Gen3Auth(refresh_file=API_KEY_FILEPATH) jobs = Gen3Jobs(auth_provider=auth) - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() job_output = loop.run_until_complete( jobs.async_run_job_and_wait(job_name=DBGAP_METADATA_JOB, job_input=JOB_INPUT) diff --git a/poetry.lock b/poetry.lock index 2f95ba983..4a082eedf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -162,11 +162,11 @@ resolved_reference = "bdfdeb05e45407e839fd954ce6d195d847cd8024" [[package]] name = "certifi" -version = "2021.10.8" +version = "2022.5.18.1" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" [[package]] name = "cffi" @@ -208,14 +208,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "coverage" -version = "6.3.3" +version = "6.4.1" description = "Code coverage measurement for Python" category = "dev" optional = false python-versions = ">=3.7" [package.dependencies] -tomli = {version = "*", optional = true, markers = "extra == \"toml\""} +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} [package.extras] toml = ["tomli"] @@ -257,15 +257,15 @@ dev = ["pytest (>=6.2.3)", "ipython", "mypy (>=0.710)", "hypothesis", "portray", [[package]] name = "dictionaryutils" -version = "3.0.2" -description = "" +version = "3.4.4" +description = "Python wrapper and metaschema for datadictionary." category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6,<3.10" [package.dependencies] -cdislogging = ">=1.0,<2.0" -jsonschema = ">=3.2,<4.0" +cdislogging = ">=1.0.0,<2.0.0" +jsonschema = ">=2.5,<4" PyYAML = ">=5.1,<6.0" requests = ">=2.18,<3.0" @@ -321,11 +321,11 @@ requests = ">=2.23.0,<3.0.0" [[package]] name = "fastavro" -version = "1.4.11" +version = "1.4.1" description = "Fast read/write of AVRO files" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.6" [package.extras] codecs = ["python-snappy", "zstandard", "lz4"] @@ -463,6 +463,21 @@ category = "main" optional = false python-versions = ">=3.5" +[[package]] +name = "importlib-metadata" +version = "1.7.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] + [[package]] name = "indexclient" version = "2.1.0" @@ -561,19 +576,19 @@ python-versions = ">=3.7" [[package]] name = "marshmallow" -version = "3.15.0" +version = "3.16.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." category = "main" optional = false python-versions = ">=3.7" [package.dependencies] -packaging = "*" +packaging = ">=17.0" [package.extras] -dev = ["pytest", "pytz", "simplejson", "mypy (==0.940)", "flake8 (==4.0.1)", "flake8-bugbear (==22.1.11)", "pre-commit (>=2.4,<3.0)", "tox"] -docs = ["sphinx (==4.4.0)", "sphinx-issues (==3.0.1)", "alabaster (==0.7.12)", "sphinx-version-warning (==1.1.2)", "autodocsumm (==0.2.7)"] -lint = ["mypy (==0.940)", "flake8 (==4.0.1)", "flake8-bugbear (==22.1.11)", "pre-commit (>=2.4,<3.0)"] +dev = ["pytest", "pytz", "simplejson", "mypy (==0.960)", "flake8 (==4.0.1)", "flake8-bugbear (==22.4.25)", "pre-commit (>=2.4,<3.0)", "tox"] +docs = ["sphinx (==4.5.0)", "sphinx-issues (==3.0.1)", "alabaster (==0.7.12)", "sphinx-version-warning (==1.1.2)", "autodocsumm (==0.2.8)"] +lint = ["mypy (==0.960)", "flake8 (==4.0.1)", "flake8-bugbear (==22.4.25)", "pre-commit (>=2.4,<3.0)"] tests = ["pytest", "pytz", "simplejson"] [[package]] @@ -605,7 +620,7 @@ python-versions = "*" [[package]] name = "numpy" -version = "1.22.3" +version = "1.22.4" description = "NumPy is the fundamental package for array computing with Python." category = "main" optional = false @@ -707,17 +722,20 @@ diagrams = ["railroad-diagrams", "jinja2"] [[package]] name = "pypfb" -version = "0.5.0" +version = "0.5.18" description = "Python SDK for PFB format" category = "main" optional = false -python-versions = ">=3.6,<4.0" +python-versions = ">=3.6.1,<3.10" [package.dependencies] +aiohttp = ">=3.6.3,<4.0.0" click = ">=7.1.2,<8.0.0" -dictionaryutils = "<=3.0.2" -fastavro = ">=1.0.0,<2.0.0" +dictionaryutils = ">=3.4.3,<4.0.0" +fastavro = "<=1.4.1" gdcdictionary = ">=1.2.0,<2.0.0" +gen3 = ">=4.2.0,<5.0.0" +importlib_metadata = {version = ">=1.3.0,<2.0.0", markers = "python_version <= \"3.9\""} python-json-logger = ">=0.1.11,<0.2.0" PyYAML = ">=5.3.1,<6.0.0" @@ -810,20 +828,20 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [[package]] name = "requests" -version = "2.27.1" +version = "2.28.0" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} -idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} +charset-normalizer = ">=2.0.0,<2.1.0" +idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] [[package]] @@ -1007,10 +1025,22 @@ python-versions = ">=3.6" idna = ">=2.0" multidict = ">=4.0" +[[package]] +name = "zipp" +version = "3.8.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] + [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.10" -content-hash = "f70393fdeaaedc8c698057876d4cdc294680a3f3696c097308f778b52531f60b" +content-hash = "8701d02f46e89faeee57ec0a1ac33b7ff7acce46779aeb57a9c7ecc39e5959c3" [metadata.files] aiofiles = [ @@ -1138,8 +1168,8 @@ cdislogging = [ ] cdisutilstest = [] certifi = [ - {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, - {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, + {file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"}, + {file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"}, ] cffi = [ {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, @@ -1206,47 +1236,47 @@ colorama = [ {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ - {file = "coverage-6.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df32ee0f4935a101e4b9a5f07b617d884a531ed5666671ff6ac66d2e8e8246d8"}, - {file = "coverage-6.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:75b5dbffc334e0beb4f6c503fb95e6d422770fd2d1b40a64898ea26d6c02742d"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:114944e6061b68a801c5da5427b9173a0dd9d32cd5fcc18a13de90352843737d"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ab88a01cd180b5640ccc9c47232e31924d5f9967ab7edd7e5c91c68eee47a69"}, - {file = "coverage-6.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad8f9068f5972a46d50fe5f32c09d6ee11da69c560fcb1b4c3baea246ca4109b"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4cd696aa712e6cd16898d63cf66139dc70d998f8121ab558f0e1936396dbc579"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c1a9942e282cc9d3ed522cd3e3cab081149b27ea3bda72d6f61f84eaf88c1a63"}, - {file = "coverage-6.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c06455121a089252b5943ea682187a4e0a5cf0a3fb980eb8e7ce394b144430a9"}, - {file = "coverage-6.3.3-cp310-cp310-win32.whl", hash = "sha256:cb5311d6ccbd22578c80028c5e292a7ab9adb91bd62c1982087fad75abe2e63d"}, - {file = "coverage-6.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:6d4a6f30f611e657495cc81a07ff7aa8cd949144e7667c5d3e680d73ba7a70e4"}, - {file = "coverage-6.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:79bf405432428e989cad7b8bc60581963238f7645ae8a404f5dce90236cc0293"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:338c417613f15596af9eb7a39353b60abec9d8ce1080aedba5ecee6a5d85f8d3"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db094a6a4ae6329ed322a8973f83630b12715654c197dd392410400a5bfa1a73"}, - {file = "coverage-6.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1414e8b124611bf4df8d77215bd32cba6e3425da8ce9c1f1046149615e3a9a31"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:93b16b08f94c92cab88073ffd185070cdcb29f1b98df8b28e6649145b7f2c90d"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fbc86ae8cc129c801e7baaafe3addf3c8d49c9c1597c44bdf2d78139707c3c62"}, - {file = "coverage-6.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b5ba058610e8289a07db2a57bce45a1793ec0d3d11db28c047aae2aa1a832572"}, - {file = "coverage-6.3.3-cp37-cp37m-win32.whl", hash = "sha256:8329635c0781927a2c6ae068461e19674c564e05b86736ab8eb29c420ee7dc20"}, - {file = "coverage-6.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:e5af1feee71099ae2e3b086ec04f57f9950e1be9ecf6c420696fea7977b84738"}, - {file = "coverage-6.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e814a4a5a1d95223b08cdb0f4f57029e8eab22ffdbae2f97107aeef28554517e"}, - {file = "coverage-6.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:61f4fbf3633cb0713437291b8848634ea97f89c7e849c2be17a665611e433f53"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3401b0d2ed9f726fadbfa35102e00d1b3547b73772a1de5508ef3bdbcb36afe7"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8586b177b4407f988731eb7f41967415b2197f35e2a6ee1a9b9b561f6323c8e9"}, - {file = "coverage-6.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:892e7fe32191960da559a14536768a62e83e87bbb867e1b9c643e7e0fbce2579"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:afb03f981fadb5aed1ac6e3dd34f0488e1a0875623d557b6fad09b97a942b38a"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cbe91bc84be4e5ef0b1480d15c7b18e29c73bdfa33e07d3725da7d18e1b0aff2"}, - {file = "coverage-6.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:91502bf27cbd5c83c95cfea291ef387469f2387508645602e1ca0fd8a4ba7548"}, - {file = "coverage-6.3.3-cp38-cp38-win32.whl", hash = "sha256:c488db059848702aff30aa1d90ef87928d4e72e4f00717343800546fdbff0a94"}, - {file = "coverage-6.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6534fcdfb5c503affb6b1130db7b5bfc8a0f77fa34880146f7a5c117987d0"}, - {file = "coverage-6.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc692c9ee18f0dd3214843779ba6b275ee4bb9b9a5745ba64265bce911aefd1a"}, - {file = "coverage-6.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:462105283de203df8de58a68c1bb4ba2a8a164097c2379f664fa81d6baf94b81"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc972d829ad5ef4d4c5fcabd2bbe2add84ce8236f64ba1c0c72185da3a273130"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:06f54765cdbce99901871d50fe9f41d58213f18e98b170a30ca34f47de7dd5e8"}, - {file = "coverage-6.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7835f76a081787f0ca62a53504361b3869840a1620049b56d803a8cb3a9eeea3"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:6f5fee77ec3384b934797f1873758f796dfb4f167e1296dc00f8b2e023ce6ee9"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:baa8be8aba3dd1e976e68677be68a960a633a6d44c325757aefaa4d66175050f"}, - {file = "coverage-6.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4d06380e777dd6b35ee936f333d55b53dc4a8271036ff884c909cf6e94be8b6c"}, - {file = "coverage-6.3.3-cp39-cp39-win32.whl", hash = "sha256:f8cabc5fd0091976ab7b020f5708335033e422de25e20ddf9416bdce2b7e07d8"}, - {file = "coverage-6.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c9441d57b0963cf8340268ad62fc83de61f1613034b79c2b1053046af0c5284"}, - {file = "coverage-6.3.3-pp36.pp37.pp38-none-any.whl", hash = "sha256:d522f1dc49127eab0bfbba4e90fa068ecff0899bbf61bf4065c790ddd6c177fe"}, - {file = "coverage-6.3.3.tar.gz", hash = "sha256:2781c43bffbbec2b8867376d4d61916f5e9c4cc168232528562a61d1b4b01879"}, + {file = "coverage-6.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f1d5aa2703e1dab4ae6cf416eb0095304f49d004c39e9db1d86f57924f43006b"}, + {file = "coverage-6.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4ce1b258493cbf8aec43e9b50d89982346b98e9ffdfaae8ae5793bc112fb0068"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c4e737f60c6936460c5be330d296dd5b48b3963f48634c53b3f7deb0f34ec4"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e65ef149028516c6d64461b95a8dbcfce95cfd5b9eb634320596173332ea84"}, + {file = "coverage-6.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69718750eaae75efe506406c490d6fc5a6161d047206cc63ce25527e8a3adad"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e57816f8ffe46b1df8f12e1b348f06d164fd5219beba7d9433ba79608ef011cc"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749"}, + {file = "coverage-6.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ab269400706fab15981fd4bd5080c56bd5cc07c3bccb86aab5e1d5a88dc8f4"}, + {file = "coverage-6.4.1-cp310-cp310-win32.whl", hash = "sha256:a7f3049243783df2e6cc6deafc49ea123522b59f464831476d3d1448e30d72df"}, + {file = "coverage-6.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2ddcac99b2d2aec413e36d7a429ae9ebcadf912946b13ffa88e7d4c9b712d6"}, + {file = "coverage-6.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fb73e0011b8793c053bfa85e53129ba5f0250fdc0392c1591fd35d915ec75c46"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87f4f3df85aa39da00fd3ec4b5abeb7407e82b68c7c5ad181308b0e2526da5d4"}, + {file = "coverage-6.4.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:961e2fb0680b4f5ad63234e0bf55dfb90d302740ae9c7ed0120677a94a1590cb"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cec3a0f75c8f1031825e19cd86ee787e87cf03e4fd2865c79c057092e69e3a3b"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:129cd05ba6f0d08a766d942a9ed4b29283aff7b2cccf5b7ce279d50796860bb3"}, + {file = "coverage-6.4.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:bf5601c33213d3cb19d17a796f8a14a9eaa5e87629a53979a5981e3e3ae166f6"}, + {file = "coverage-6.4.1-cp37-cp37m-win32.whl", hash = "sha256:269eaa2c20a13a5bf17558d4dc91a8d078c4fa1872f25303dddcbba3a813085e"}, + {file = "coverage-6.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f02cbbf8119db68455b9d763f2f8737bb7db7e43720afa07d8eb1604e5c5ae28"}, + {file = "coverage-6.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54"}, + {file = "coverage-6.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:145f296d00441ca703a659e8f3eb48ae39fb083baba2d7ce4482fb2723e050d9"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d44996140af8b84284e5e7d398e589574b376fb4de8ccd28d82ad8e3bea13"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2bd9a6fc18aab8d2e18f89b7ff91c0f34ff4d5e0ba0b33e989b3cd4194c81fd9"}, + {file = "coverage-6.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3384f2a3652cef289e38100f2d037956194a837221edd520a7ee5b42d00cc605"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9b3e07152b4563722be523e8cd0b209e0d1a373022cfbde395ebb6575bf6790d"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1480ff858b4113db2718848d7b2d1b75bc79895a9c22e76a221b9d8d62496428"}, + {file = "coverage-6.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:865d69ae811a392f4d06bde506d531f6a28a00af36f5c8649684a9e5e4a85c83"}, + {file = "coverage-6.4.1-cp38-cp38-win32.whl", hash = "sha256:664a47ce62fe4bef9e2d2c430306e1428ecea207ffd68649e3b942fa8ea83b0b"}, + {file = "coverage-6.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:26dff09fb0d82693ba9e6231248641d60ba606150d02ed45110f9ec26404ed1c"}, + {file = "coverage-6.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9c80df769f5ec05ad21ea34be7458d1dc51ff1fb4b2219e77fe24edf462d6df"}, + {file = "coverage-6.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:39ee53946bf009788108b4dd2894bf1349b4e0ca18c2016ffa7d26ce46b8f10d"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5b66caa62922531059bc5ac04f836860412f7f88d38a476eda0a6f11d4724f4"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd180ed867e289964404051a958f7cccabdeed423f91a899829264bb7974d3d3"}, + {file = "coverage-6.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84631e81dd053e8a0d4967cedab6db94345f1c36107c71698f746cb2636c63e3"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:8c08da0bd238f2970230c2a0d28ff0e99961598cb2e810245d7fc5afcf1254e8"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d42c549a8f41dc103a8004b9f0c433e2086add8a719da00e246e17cbe4056f72"}, + {file = "coverage-6.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:309ce4a522ed5fca432af4ebe0f32b21d6d7ccbb0f5fcc99290e71feba67c264"}, + {file = "coverage-6.4.1-cp39-cp39-win32.whl", hash = "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9"}, + {file = "coverage-6.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:342d4aefd1c3e7f620a13f4fe563154d808b69cccef415415aece4c786665397"}, + {file = "coverage-6.4.1-pp36.pp37.pp38-none-any.whl", hash = "sha256:4803e7ccf93230accb928f3a68f00ffa80a88213af98ed338a57ad021ef06815"}, + {file = "coverage-6.4.1.tar.gz", hash = "sha256:4321f075095a096e70aff1d002030ee612b65a205a0a0f5b815280d5dc58100c"}, ] cryptography = [ {file = "cryptography-37.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:ef15c2df7656763b4ff20a9bc4381d8352e6640cfeb95c2972c38ef508e75181"}, @@ -1277,8 +1307,7 @@ dataclasses-json = [ {file = "dataclasses_json-0.5.7-py3-none-any.whl", hash = "sha256:bc285b5f892094c3a53d558858a88553dd6a61a11ab1a8128a0e554385dcc5dd"}, ] dictionaryutils = [ - {file = "dictionaryutils-3.0.2-py3.6.egg", hash = "sha256:5a592bb65b7a6c7e96d6c367f2efdd3192d0d6ee19cb54808ddd65f271305170"}, - {file = "dictionaryutils-3.0.2.tar.gz", hash = "sha256:90443a3ed20409198bbd605164672f420d8054117f65c2f6d84ac02969f7e42c"}, + {file = "dictionaryutils-3.4.4.tar.gz", hash = "sha256:b268196a58b100082cdda9e9a8a7f96fd637fc352e3572b138c19d1d8a98ab6f"}, ] doiclient = [] dosclient = [] @@ -1286,22 +1315,20 @@ drsclient = [ {file = "drsclient-0.2.1.tar.gz", hash = "sha256:c6bc62b19beef9276cb2d3757e8b866bc4ee1c04ff105832ee1924798680588b"}, ] fastavro = [ - {file = "fastavro-1.4.11-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:44f01008f95d685edacc4b10366c755d25612df00924349f7d34a29f08522ce3"}, - {file = "fastavro-1.4.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18f5e736d12e67348f253da8a332d7c3b483ca04f2b6e772befa79d1a46bac9d"}, - {file = "fastavro-1.4.11-cp310-cp310-win_amd64.whl", hash = "sha256:8dca11bc3191cd7de0a3c4b76a70dac493356a219e96ebcde0def1f06faddef7"}, - {file = "fastavro-1.4.11-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:7a2a0bf03686f9d860e8f8476be000f5b3e6cc9af6853dbabab2ef9cfa5dc3a0"}, - {file = "fastavro-1.4.11-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c17e3decfac260e1be4d02d1903d2483eec2f3ce7f92c9b808a0f6a81572c4b"}, - {file = "fastavro-1.4.11-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:19ba25c6529f50722a7618cc4ca24c7d265def57fd9f94e4e554e1df8cce38d2"}, - {file = "fastavro-1.4.11-cp37-cp37m-win_amd64.whl", hash = "sha256:ceaba04da9419f40899a670eb62eb373a127b511bb8e3ae4f6f1f23ec49bd0e4"}, - {file = "fastavro-1.4.11-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:732eab3a1ae5d2c3f4b52e747c55bcc41c4df0eb7e8a395038080741a3c0a934"}, - {file = "fastavro-1.4.11-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c03d3c802b71f44e7b3442abae961bba996258244bd222b242ad1e5cb7754e57"}, - {file = "fastavro-1.4.11-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7cb7475a9b25b9f8aebe7eb756dafedd0369434571062f3883d894281befd7c"}, - {file = "fastavro-1.4.11-cp38-cp38-win_amd64.whl", hash = "sha256:ce0776f54591aef90bcd02bd919964abe4c2ad2a10a4336c3a1b66cef289b41c"}, - {file = "fastavro-1.4.11-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:621e72cc365c9539d7590e7b43e48a62e6bfb4c2de7c16837fed54d113d7312c"}, - {file = "fastavro-1.4.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:842b25782f911ee8c626f9d9fedc2ef01aeac272536fe90ee6d45b2ae7cdb024"}, - {file = "fastavro-1.4.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8491bfcba25c9d661289f884688e5a4f56f2ee389a240d0ad02692495a9a087"}, - {file = "fastavro-1.4.11-cp39-cp39-win_amd64.whl", hash = "sha256:c94130a8c8d80073eb0276844915aa5e928ae322024e76dc57943542ccda211c"}, - {file = "fastavro-1.4.11.tar.gz", hash = "sha256:7c64332ad52de0134be9a933ca986514c3ff85c63d54bc5398c31f0498ac1820"}, + {file = "fastavro-1.4.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:3f99237de0f853f083a0f9929f54155b408a5c4b04fcfa8c59e589aa853f2111"}, + {file = "fastavro-1.4.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18ed93c24ba96ed0e6cacf50170805188cc53a5e50c40fc94316387a98aa497b"}, + {file = "fastavro-1.4.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c298d2b2389049dfc6a06326a0a2f9809c7045e8cadb82b9e7e948fd32270547"}, + {file = "fastavro-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:948e69da16f4bf20bea65805ea210d793eae55f5f24f8be2c4d18c2a773aaf7b"}, + {file = "fastavro-1.4.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:a4b6c2b2d126bf6246bead79cba43e17f07a1b99a25ccceb315c6e75ab7c9d39"}, + {file = "fastavro-1.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:453676a26e99f2f3af7f57c4236ceeee4e453ab7b9bd5f09e9c89bad5e572c78"}, + {file = "fastavro-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5bd8a134daff2ea5ef0d72a528ca42dfe6c01deac7103dedf77d1a55936a981e"}, + {file = "fastavro-1.4.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:a8c5df1aa6c29409bbfe571504e87bf23553c88083a83febb42a74ec0cabe2ba"}, + {file = "fastavro-1.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:741cd757b7789e6ab821a1de02c1e18dfada417e1cfccae3f20dc2aadd6654fc"}, + {file = "fastavro-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:6d7d4032ecb28bef3dd41e8c91c986df351e3323f526c901d1cbb53425617756"}, + {file = "fastavro-1.4.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:72c81690cef6ed9c87a146eaf9f608150bf78fd537a7e796780381b4a7baabb8"}, + {file = "fastavro-1.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cad3ecbac1fe1d319c617ff01104639795020cb72faf7e30ebe802c1d60ec915"}, + {file = "fastavro-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:3e804c4fc9875314aa41901055941b199f87aeb1c880cc6fe3ad258fa08b24c6"}, + {file = "fastavro-1.4.1.tar.gz", hash = "sha256:237a741668316a2aadb14ba0532666a305dd14b4043aace89bcb0c6419c08162"}, ] flask = [ {file = "Flask-2.0.3-py3-none-any.whl", hash = "sha256:59da8a3170004800a2837844bfa84d49b022550616070f7cb1a659682b2e7c9f"}, @@ -1397,6 +1424,10 @@ idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] +importlib-metadata = [ + {file = "importlib_metadata-1.7.0-py2.py3-none-any.whl", hash = "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070"}, + {file = "importlib_metadata-1.7.0.tar.gz", hash = "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83"}, +] indexclient = [ {file = "indexclient-2.1.0.tar.gz", hash = "sha256:777476eb97febfcd663b7f9454fba5151dee4d17254bfa0dfb5ff040c59b8532"}, ] @@ -1460,8 +1491,8 @@ markupsafe = [ {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] marshmallow = [ - {file = "marshmallow-3.15.0-py3-none-any.whl", hash = "sha256:ff79885ed43b579782f48c251d262e062bce49c65c52412458769a4fb57ac30f"}, - {file = "marshmallow-3.15.0.tar.gz", hash = "sha256:2aaaab4f01ef4f5a011a21319af9fce17ab13bf28a026d1252adab0e035648d5"}, + {file = "marshmallow-3.16.0-py3-none-any.whl", hash = "sha256:53a1e0ee69f79e1f3e80d17393b25cfc917eda52f859e8183b4af72c3390c1f1"}, + {file = "marshmallow-3.16.0.tar.gz", hash = "sha256:a762c1d8b2bcb0e5c8e964850d03f9f3bffd6a12b626f3c14b9d6b1841999af5"}, ] marshmallow-enum = [ {file = "marshmallow-enum-1.5.1.tar.gz", hash = "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58"}, @@ -1533,26 +1564,28 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] numpy = [ - {file = "numpy-1.22.3-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:92bfa69cfbdf7dfc3040978ad09a48091143cffb778ec3b03fa170c494118d75"}, - {file = "numpy-1.22.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8251ed96f38b47b4295b1ae51631de7ffa8260b5b087808ef09a39a9d66c97ab"}, - {file = "numpy-1.22.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48a3aecd3b997bf452a2dedb11f4e79bc5bfd21a1d4cc760e703c31d57c84b3e"}, - {file = "numpy-1.22.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3bae1a2ed00e90b3ba5f7bd0a7c7999b55d609e0c54ceb2b076a25e345fa9f4"}, - {file = "numpy-1.22.3-cp310-cp310-win32.whl", hash = "sha256:f950f8845b480cffe522913d35567e29dd381b0dc7e4ce6a4a9f9156417d2430"}, - {file = "numpy-1.22.3-cp310-cp310-win_amd64.whl", hash = "sha256:08d9b008d0156c70dc392bb3ab3abb6e7a711383c3247b410b39962263576cd4"}, - {file = "numpy-1.22.3-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:201b4d0552831f7250a08d3b38de0d989d6f6e4658b709a02a73c524ccc6ffce"}, - {file = "numpy-1.22.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f8c1f39caad2c896bc0018f699882b345b2a63708008be29b1f355ebf6f933fe"}, - {file = "numpy-1.22.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:568dfd16224abddafb1cbcce2ff14f522abe037268514dd7e42c6776a1c3f8e5"}, - {file = "numpy-1.22.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ca688e1b9b95d80250bca34b11a05e389b1420d00e87a0d12dc45f131f704a1"}, - {file = "numpy-1.22.3-cp38-cp38-win32.whl", hash = "sha256:e7927a589df200c5e23c57970bafbd0cd322459aa7b1ff73b7c2e84d6e3eae62"}, - {file = "numpy-1.22.3-cp38-cp38-win_amd64.whl", hash = "sha256:07a8c89a04997625236c5ecb7afe35a02af3896c8aa01890a849913a2309c676"}, - {file = "numpy-1.22.3-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2c10a93606e0b4b95c9b04b77dc349b398fdfbda382d2a39ba5a822f669a0123"}, - {file = "numpy-1.22.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fade0d4f4d292b6f39951b6836d7a3c7ef5b2347f3c420cd9820a1d90d794802"}, - {file = "numpy-1.22.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bfb1bb598e8229c2d5d48db1860bcf4311337864ea3efdbe1171fb0c5da515d"}, - {file = "numpy-1.22.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97098b95aa4e418529099c26558eeb8486e66bd1e53a6b606d684d0c3616b168"}, - {file = "numpy-1.22.3-cp39-cp39-win32.whl", hash = "sha256:fdf3c08bce27132395d3c3ba1503cac12e17282358cb4bddc25cc46b0aca07aa"}, - {file = "numpy-1.22.3-cp39-cp39-win_amd64.whl", hash = "sha256:639b54cdf6aa4f82fe37ebf70401bbb74b8508fddcf4797f9fe59615b8c5813a"}, - {file = "numpy-1.22.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c34ea7e9d13a70bf2ab64a2532fe149a9aced424cd05a2c4ba662fd989e3e45f"}, - {file = "numpy-1.22.3.zip", hash = "sha256:dbc7601a3b7472d559dc7b933b18b4b66f9aa7452c120e87dfb33d02008c8a18"}, + {file = "numpy-1.22.4-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ba9ead61dfb5d971d77b6c131a9dbee62294a932bf6a356e48c75ae684e635b3"}, + {file = "numpy-1.22.4-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:1ce7ab2053e36c0a71e7a13a7475bd3b1f54750b4b433adc96313e127b870887"}, + {file = "numpy-1.22.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7228ad13744f63575b3a972d7ee4fd61815b2879998e70930d4ccf9ec721dce0"}, + {file = "numpy-1.22.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43a8ca7391b626b4c4fe20aefe79fec683279e31e7c79716863b4b25021e0e74"}, + {file = "numpy-1.22.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a911e317e8c826ea632205e63ed8507e0dc877dcdc49744584dfc363df9ca08c"}, + {file = "numpy-1.22.4-cp310-cp310-win32.whl", hash = "sha256:9ce7df0abeabe7fbd8ccbf343dc0db72f68549856b863ae3dd580255d009648e"}, + {file = "numpy-1.22.4-cp310-cp310-win_amd64.whl", hash = "sha256:3e1ffa4748168e1cc8d3cde93f006fe92b5421396221a02f2274aab6ac83b077"}, + {file = "numpy-1.22.4-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:59d55e634968b8f77d3fd674a3cf0b96e85147cd6556ec64ade018f27e9479e1"}, + {file = "numpy-1.22.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c1d937820db6e43bec43e8d016b9b3165dcb42892ea9f106c70fb13d430ffe72"}, + {file = "numpy-1.22.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4c5d5eb2ec8da0b4f50c9a843393971f31f1d60be87e0fb0917a49133d257d6"}, + {file = "numpy-1.22.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64f56fc53a2d18b1924abd15745e30d82a5782b2cab3429aceecc6875bd5add0"}, + {file = "numpy-1.22.4-cp38-cp38-win32.whl", hash = "sha256:fb7a980c81dd932381f8228a426df8aeb70d59bbcda2af075b627bbc50207cba"}, + {file = "numpy-1.22.4-cp38-cp38-win_amd64.whl", hash = "sha256:e96d7f3096a36c8754207ab89d4b3282ba7b49ea140e4973591852c77d09eb76"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:4c6036521f11a731ce0648f10c18ae66d7143865f19f7299943c985cdc95afb5"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b89bf9b94b3d624e7bb480344e91f68c1c6c75f026ed6755955117de00917a7c"}, + {file = "numpy-1.22.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2d487e06ecbf1dc2f18e7efce82ded4f705f4bd0cd02677ffccfb39e5c284c7e"}, + {file = "numpy-1.22.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3eb268dbd5cfaffd9448113539e44e2dd1c5ca9ce25576f7c04a5453edc26fa"}, + {file = "numpy-1.22.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37431a77ceb9307c28382c9773da9f306435135fae6b80b62a11c53cfedd8802"}, + {file = "numpy-1.22.4-cp39-cp39-win32.whl", hash = "sha256:cc7f00008eb7d3f2489fca6f334ec19ca63e31371be28fd5dad955b16ec285bd"}, + {file = "numpy-1.22.4-cp39-cp39-win_amd64.whl", hash = "sha256:f0725df166cf4785c0bc4cbfb320203182b1ecd30fee6e541c8752a92df6aa32"}, + {file = "numpy-1.22.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0791fbd1e43bf74b3502133207e378901272f3c156c4df4954cad833b1380207"}, + {file = "numpy-1.22.4.zip", hash = "sha256:425b390e4619f58d8526b3dcf656dde069133ae5c240229821f01b5f44ea07af"}, ] packaging = [ {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, @@ -1615,8 +1648,8 @@ pyparsing = [ {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, ] pypfb = [ - {file = "pypfb-0.5.0-py3-none-any.whl", hash = "sha256:db654407276bb347d9f2e5c640ade3d32cd70e46f5aeec0f8dce1a963003d7ff"}, - {file = "pypfb-0.5.0.tar.gz", hash = "sha256:319f16cec65bb40b27d9e318685061a4abe27b15a1d688db6b3f7d19e52ade97"}, + {file = "pypfb-0.5.18-py3-none-any.whl", hash = "sha256:eb725ad559ae66770da482bb894f09ef589bd77689d27106f2c70ddaa0527572"}, + {file = "pypfb-0.5.18.tar.gz", hash = "sha256:707c92515655a9ad8a1bba00a8d12ac80ce18f0c4e119c07ccbef32f9650a318"}, ] pyreadline3 = [ {file = "pyreadline3-3.4.1-py3-none-any.whl", hash = "sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb"}, @@ -1696,8 +1729,8 @@ pyyaml = [ {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, ] requests = [ - {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"}, - {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"}, + {file = "requests-2.28.0-py3-none-any.whl", hash = "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f"}, + {file = "requests-2.28.0.tar.gz", hash = "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b"}, ] requests-mock = [ {file = "requests-mock-1.9.3.tar.gz", hash = "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba"}, @@ -1858,3 +1891,7 @@ yarl = [ {file = "yarl-1.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:797c2c412b04403d2da075fb93c123df35239cd7b4cc4e0cd9e5839b73f52c58"}, {file = "yarl-1.7.2.tar.gz", hash = "sha256:45399b46d60c253327a460e99856752009fcee5f5d3c80b2f7c0cae1c38d56dd"}, ] +zipp = [ + {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, + {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, +] diff --git a/pyproject.toml b/pyproject.toml index 2252e62b0..04dd486fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "gen3" homepage = "https://gen3.org/" -version = "4.10.0" +version = "4.10.1" description = "Gen3 CLI and Python SDK" authors = ["Center for Translational Data Science at the University of Chicago "] license = "Apache-2.0" @@ -28,12 +28,12 @@ click = "*" jsonschema = "*" dataclasses-json = "*" pypfb = "<1.0.0" -fastavro = "^1.4.7" tqdm = ">=4.61.2" humanfriendly ="*" python-dateutil = "*" aiofiles = "^0.8.0" pandas = "^1.4.2" +httpx = "*" [tool.poetry.dev-dependencies] pytest = ">=6.0.0" diff --git a/tests/conftest.py b/tests/conftest.py index 58ef16a4e..0118e8f26 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -154,7 +154,6 @@ def auth_request(*args, **kwargs): yield MockServer(port=port) indexd.terminate() - wait_for_indexd_not_alive(port) @pytest.fixture diff --git a/tests/test_manifests.py b/tests/test_manifests.py index 86e1e742a..7a5de1187 100644 --- a/tests/test_manifests.py +++ b/tests/test_manifests.py @@ -13,6 +13,7 @@ ) from gen3.tools.utils import get_and_verify_fileinfos_from_tsv_manifest +from gen3.utils import get_or_create_event_loop_for_thread CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -66,9 +67,8 @@ def test_verify_manifest(mock_index): NOTE: records in indexd are mocked """ mock_index.return_value.async_get_record.side_effect = _async_mock_get_guid - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() loop.run_until_complete( async_verify_object_manifest( "http://localhost", @@ -165,9 +165,8 @@ def test_download_manifest(monkeypatch, gen3_index): # mock_index.return_value.get_stats.return_value = gen3_index.get("/_stats") monkeypatch.setattr(download_manifest, "INDEXD_RECORD_PAGE_SIZE", 2) - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() loop.run_until_complete( async_download_object_manifest( "http://localhost:8001", @@ -293,9 +292,8 @@ def test_download_manifest_with_input_manifest(monkeypatch, gen3_index): ) monkeypatch.setattr(download_manifest, "INDEXD_RECORD_PAGE_SIZE", 2) - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() loop.run_until_complete( async_download_object_manifest( "http://localhost:8001", @@ -431,9 +429,8 @@ def test_download_manifest_with_invalid_input_manifest(monkeypatch, gen3_index): ) monkeypatch.setattr(download_manifest, "INDEXD_RECORD_PAGE_SIZE", 2) - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) + loop = get_or_create_event_loop_for_thread() # ensure error is raised try: loop.run_until_complete(