From 87f620b7d276f30d19667442a923a4f1d13884fe Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 29 May 2020 11:51:10 -0500 Subject: [PATCH 01/39] revert to authlib==0.14.2 --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 75110b4e8..7fdb7d2a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,9 @@ WORKDIR /sheepdog RUN python -m pip install --upgrade pip \ && python -m pip install --upgrade setuptools \ && pip --version \ - && pip install -r requirements.txt + && pip install -r requirements.txt \ + && pip uninstall authlib -y \ + && pip install authlib==0.14.2 RUN mkdir -p /var/www/sheepdog \ && mkdir /run/ngnix/ \ From 51516b32824cbc172a5c24ce00080bb67c078d50 Mon Sep 17 00:00:00 2001 From: grugna Date: Wed, 10 Jun 2020 18:40:34 -0500 Subject: [PATCH 02/39] add configurable node name/model for granular auth --- sheepdog/api.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sheepdog/api.py b/sheepdog/api.py index 79d9e4937..c544ced2d 100644 --- a/sheepdog/api.py +++ b/sheepdog/api.py @@ -1,5 +1,6 @@ import os import sys +import importlib from flask import Flask, jsonify from psqlgraph import PsqlGraphDriver @@ -157,6 +158,17 @@ def app_init(app): app.logger.info("Using default Arborist base URL") app.auth = ArboristClient() + app.node_authz_entity_name = os.environ.get("AUTHZ_ENTITY_NAME", None) + if app.node_authz_entity_name: + full_module_name = "datamodelutils.models" + mymodule = importlib.import_module(full_module_name) + for i in dir(mymodule): + if i.lower() == app.node_authz_entity_name.lower(): + attribute = getattr(mymodule, i) + app.node_authz_entity = attribute + else: + app.node_authz_entity = None + app = Flask(__name__) From 78fe8ec131af91fae2cff7f029ab9e2d7c492325 Mon Sep 17 00:00:00 2001 From: grugna Date: Wed, 10 Jun 2020 18:41:18 -0500 Subject: [PATCH 03/39] expand auth functions to check if user has access on the level under project --- sheepdog/auth/__init__.py | 79 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index bad753153..02cd8b284 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -15,6 +15,7 @@ import flask from sheepdog.errors import AuthNError, AuthZError +from sheepdog.globals import ROLES logger = get_logger(__name__) @@ -110,12 +111,21 @@ def authorize_and_call(*args, **kwargs): return authorize_and_call -def authorize(program, project, roles): +def authorize(program, project, roles, resources_tmp=None): resource = "/programs/{}/projects/{}".format(program, project) + + resources = [] + if resources_tmp: + for resource_tmp in resources_tmp: + resources.append(resource + resource_tmp) + else: + resources = [resource] + jwt = get_jwt_from_header() authz = flask.current_app.auth.auth_request( - jwt=jwt, service="sheepdog", methods=roles, resources=[resource] + jwt=jwt, service="sheepdog", methods=roles, resources=resources ) + if not authz: raise AuthZError("user is unauthorized") @@ -139,3 +149,68 @@ def create_resource(program, project=None): resp.error.code, resp.error.message ) ) + + +def check_resource_access(program, project, nodes): + subject_submitter_ids = [] + stop_node = flask.current_app.node_authz_entity_name + + for node in nodes: + if node.label == stop_node: + subject_submitter_ids.append({"id": node.node_id, "submitter_id": node.props.get("submitter_id", None)}) + else: + for link in node._pg_links: + tmp_dad = getattr(node, link) + nodeType = link + path_tmp = nodeType + tmp = node._pg_links[link]["dst_type"] + while tmp.label != stop_node and tmp.label != "program": + # assuming ony one parents + nodeType = list(tmp._pg_links.keys())[0] + path_tmp = path_tmp + "." + nodeType + tmp = tmp._pg_links[nodeType]["dst_type"] + # TODO double check this with deeper relationship > 2 nodes under project + tmp_dad = getattr(tmp_dad, nodeType)[0] + + if tmp.label == stop_node: + subject_submitter_ids.append({"id": tmp_dad[0].node_id, "submitter_id": tmp_dad[0].props.get("submitter_id", None)}) + + try: + resources = [ + "/{}s/{}".format(stop_node, node["submitter_id"]) + for node in subject_submitter_ids + ] + authorize(program, project, [ROLES["READ"]], resources) + except AuthZError: + return "You do not have read permission on project {} for one or more of the subjects requested" + + +def get_authorized_ids(program, project): + try: + mapping = flask.current_app.auth.auth_mapping(current_user.username) + except ArboristError as e: + logger.warn( + "Unable to retrieve auth mapping for user `{}`: {}".format(current_user.username, e) + ) + mapping = {} + + base_resource_path = "/programs/{}/projects/{}".format(program, project) + result = [resource_path for resource_path, permissions in mapping.items() if base_resource_path in resource_path] + ids = [] + + for path in result: + parts = path.strip("/").split("/") + if path != "/" and parts[0] != "programs": + continue + + if len(parts) > 6 or (len(parts) > 2 and parts[2] != "projects") or (len(parts) > 4 and (flask.current_app.node_authz_entity_name is None or flask.current_app.node_authz_entity is None or parts[4] != (flask.current_app.node_authz_entity_name + "s"))): + continue + + if len(parts) < 6: + return(None) + break + else: + ids.append(parts[5]) + + return(ids) + From 3886c9b982e0cbc5a257f2a2da8e694ff47dd976 Mon Sep 17 00:00:00 2001 From: grugna Date: Wed, 10 Jun 2020 18:44:30 -0500 Subject: [PATCH 04/39] fix bug on select entities entities from another program-project were selected with the previous code. replaced authorize_for_project with other auth function to handle the granularity --- .../blueprint/routes/views/program/project.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/sheepdog/blueprint/routes/views/program/project.py b/sheepdog/blueprint/routes/views/program/project.py index 7bf41ef79..327d1a25e 100644 --- a/sheepdog/blueprint/routes/views/program/project.py +++ b/sheepdog/blueprint/routes/views/program/project.py @@ -233,7 +233,6 @@ def get_project_dictionary_entry(program, project, entry): @utils.assert_program_exists -@auth.authorize_for_project(ROLES["READ"]) def get_entities_by_id(program, project, entity_id_string): """ Retrieve existing GDC entities by ID. @@ -267,15 +266,26 @@ def get_entities_by_id(program, project, entity_id_string): :reqheader X-Auth-Token: |reqheader_X-Auth-Token| :resheader Content-Type: |resheader_Content-Type| """ + entity_ids = entity_id_string.split(",") with flask.current_app.db.session_scope(): - nodes = flask.current_app.db.nodes().ids(entity_ids).all() + dictionary_nodes = flask.current_app.db.nodes().ids(entity_ids).props(project_id = program + "-" + project).all() + project_nodes = flask.current_app.db.nodes(models.Project).ids(entity_ids).all() + program_nodes = flask.current_app.db.nodes(models.Program).ids(entity_ids).all() + + nodes = [] + nodes.extend(dictionary_nodes) + nodes.extend(project_nodes) + nodes.extend(program_nodes) + + auth.check_resource_access(program, project, nodes) + entities = {n.node_id: n for n in nodes} missing_entities = set(entity_ids) - set(entities.keys()) if missing_entities: raise UserError( "Not found: {}".format(", ".join(missing_entities), code=404) - ) + ) return flask.jsonify({"entities": utils.create_entity_list(entities.values())}) @@ -345,7 +355,6 @@ def delete_entities(program, project, ids, to_delete=None): return delete_entities -@auth.authorize_for_project(ROLES["READ"]) def export_entities(program, project): """ Return a file with the requested entities as an attachment. From 4920522b0281d2454577fd5667936cff4a45c66b Mon Sep 17 00:00:00 2001 From: grugna Date: Wed, 10 Jun 2020 18:47:56 -0500 Subject: [PATCH 05/39] added auth function when exporting nodes to handle 1 level below project granularity --- sheepdog/utils/transforms/graph_to_doc.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index c1659092d..9409f2ca5 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -15,6 +15,7 @@ import flask import psqlgraph +from sheepdog import auth from sheepdog import dictionary from sheepdog.errors import InternalError, NotFoundError, UnsupportedError, UserError from sheepdog.globals import DELIMITERS, SUB_DELIMITERS, SUPPORTED_FORMATS @@ -487,6 +488,7 @@ def get_nodes(self, ids, with_children, without_id): .props(project_id=self.project_id) .all() ) + auth.check_resource_access(self.program, self.project, self.nodes) found_ids = {node.node_id for node in self.nodes} if not found_ids: raise NotFoundError("Unable to find {}".format(", ".join(ids))) @@ -811,6 +813,12 @@ def export_all(node_label, project_id, file_format, db, without_id): # and secondly, all the relevant properties in linked nodes. query_args = [cls] + linked_props query = session.query(*query_args).prop("project_id", project_id) + + #add filter by id the user is authorized to access + auth_ids = auth.get_authorized_ids(project_id.split('-')[0], project_id.split('-')[1]) + if auth_ids: + query = query.prop_in('submitter_id', auth_ids) + # Join the related node tables using the links. for link in cls._pg_links.values(): query = query.outerjoin(link["edge_out"]).outerjoin(link["dst_type"]) From 6668bb7d23660d3957b4f098e6229dc7e3bf57c9 Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 17 Aug 2020 16:00:32 -0500 Subject: [PATCH 06/39] remove patch since main dependency has been fixed --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7fdb7d2a8..e8a824cd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,9 +15,9 @@ WORKDIR /sheepdog RUN python -m pip install --upgrade pip \ && python -m pip install --upgrade setuptools \ && pip --version \ - && pip install -r requirements.txt \ - && pip uninstall authlib -y \ - && pip install authlib==0.14.2 + && pip install -r requirements.txt + # && pip uninstall authlib -y \ + # && pip install authlib==0.14.2 RUN mkdir -p /var/www/sheepdog \ && mkdir /run/ngnix/ \ From 76dc42b1c37e84664fda12316d7600e98e9750a1 Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 24 Sep 2020 16:46:34 -0500 Subject: [PATCH 07/39] improve create_resource in sheepdog after submission --- sheepdog/auth/__init__.py | 10 +++++++++- sheepdog/transactions/upload/__init__.py | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index 02cd8b284..f02ac4417 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -130,7 +130,11 @@ def authorize(program, project, roles, resources_tmp=None): raise AuthZError("user is unauthorized") -def create_resource(program, project=None): +def create_resource(program, project=None, extra_data=None): + logger.warn("LUCA RESOURCE ATTENTION") + logger.warn(extra_data) + + resource = "/programs/{}".format(program) if project: resource += "/projects/{}".format(project) @@ -174,6 +178,9 @@ def check_resource_access(program, project, nodes): if tmp.label == stop_node: subject_submitter_ids.append({"id": tmp_dad[0].node_id, "submitter_id": tmp_dad[0].props.get("submitter_id", None)}) + else: + logger.warn("resource not found " + node.label) + logger.warn(node) try: resources = [ @@ -185,6 +192,7 @@ def check_resource_access(program, project, nodes): return "You do not have read permission on project {} for one or more of the subjects requested" +# TEST BUT YOU NEED TO ADD ACTUAL ID LIST NOT ONLY THE ONE LISTED IN THE DB def get_authorized_ids(program, project): try: mapping = flask.current_app.auth.auth_mapping(current_user.username) diff --git a/sheepdog/transactions/upload/__init__.py b/sheepdog/transactions/upload/__init__.py index 8effbca26..563d3f863 100644 --- a/sheepdog/transactions/upload/__init__.py +++ b/sheepdog/transactions/upload/__init__.py @@ -139,9 +139,17 @@ def handle_single_transaction(role, program, project, **tx_kwargs): flask.current_app.async_pool.schedule( single_transaction_worker, transaction, *doc_args ) + + # create the resource in arborist + auth.create_resource(program, project, doc_args.data) + return flask.jsonify(response) else: response, code = single_transaction_worker(transaction, *doc_args) + + # create the resource in arborist + auth.create_resource(program, project, doc_args.data) + return flask.jsonify(response), code From 1cc80ccf2322557d71381b1fdd7aeed910a40d3e Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 24 Sep 2020 17:46:26 -0500 Subject: [PATCH 08/39] fix --- sheepdog/transactions/upload/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sheepdog/transactions/upload/__init__.py b/sheepdog/transactions/upload/__init__.py index 563d3f863..859e6b6c6 100644 --- a/sheepdog/transactions/upload/__init__.py +++ b/sheepdog/transactions/upload/__init__.py @@ -141,14 +141,14 @@ def handle_single_transaction(role, program, project, **tx_kwargs): ) # create the resource in arborist - auth.create_resource(program, project, doc_args.data) + auth.create_resource(program, project, doc_args) return flask.jsonify(response) else: response, code = single_transaction_worker(transaction, *doc_args) # create the resource in arborist - auth.create_resource(program, project, doc_args.data) + auth.create_resource(program, project, doc_args) return flask.jsonify(response), code From d9faa3c368062238d9f6b1db4c40f0b353a039de Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 25 Sep 2020 10:00:10 -0500 Subject: [PATCH 09/39] test create resource granular --- sheepdog/auth/__init__.py | 22 +++++++++++++++++++--- sheepdog/transactions/upload/__init__.py | 4 ++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index f02ac4417..e2908d00f 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -130,14 +130,30 @@ def authorize(program, project, roles, resources_tmp=None): raise AuthZError("user is unauthorized") -def create_resource(program, project=None, extra_data=None): +def create_resource(program, project=None, data=None): logger.warn("LUCA RESOURCE ATTENTION") - logger.warn(extra_data) - + logger.warn(data) # {'type': 'subject', 'persons': [{'submitter_id': 'lavefrrg'}], 'submitter_id': 'test_sub_1', 'state': 'validated'} + resource = "/programs/{}".format(program) + if project: resource += "/projects/{}".format(project) + + stop_node = flask.current_app.node_authz_entity + person_node = flask.current_app.subject_entity + logger.warn(stop_node) + logger.warn(person_node) + logger.warn(stop_node[:-1]) + logger.warn(person_node[:-1]) + + if data and data["type"] == "person": + resource += "/persons/{}".format(data["submitter_id"]) + elif data and data["type"] == "subject": + resource += "/persons/{}/subjects/{}".format(data["persons"][0]["submitter_id"], data["submitter_id"]) + logger.warn(resource) + + logger.info("Creating arborist resource {}".format(resource)) json_data = { diff --git a/sheepdog/transactions/upload/__init__.py b/sheepdog/transactions/upload/__init__.py index 859e6b6c6..e55079351 100644 --- a/sheepdog/transactions/upload/__init__.py +++ b/sheepdog/transactions/upload/__init__.py @@ -141,14 +141,14 @@ def handle_single_transaction(role, program, project, **tx_kwargs): ) # create the resource in arborist - auth.create_resource(program, project, doc_args) + auth.create_resource(program, project, doc_args[3]) return flask.jsonify(response) else: response, code = single_transaction_worker(transaction, *doc_args) # create the resource in arborist - auth.create_resource(program, project, doc_args) + auth.create_resource(program, project, doc_args[3]) return flask.jsonify(response), code From bdf10e2a6cead6821b220df15346ebdde28a5a62 Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 25 Sep 2020 10:23:55 -0500 Subject: [PATCH 10/39] collect person class at startup --- sheepdog/api.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sheepdog/api.py b/sheepdog/api.py index c544ced2d..170dbf8f8 100644 --- a/sheepdog/api.py +++ b/sheepdog/api.py @@ -158,16 +158,22 @@ def app_init(app): app.logger.info("Using default Arborist base URL") app.auth = ArboristClient() + app.node_authz_entity_name = os.environ.get("AUTHZ_ENTITY_NAME", None) + app.node_authz_entity = None + app.subject_entity = None if app.node_authz_entity_name: full_module_name = "datamodelutils.models" mymodule = importlib.import_module(full_module_name) for i in dir(mymodule): + app.logger.warn(i) + if i.lower() == "person": + attribute = getattr(mymodule, i) + app.subject_entity = attribute if i.lower() == app.node_authz_entity_name.lower(): attribute = getattr(mymodule, i) app.node_authz_entity = attribute - else: - app.node_authz_entity = None + app = Flask(__name__) From be794e46c23c24b33409ee7efec8c8ca78fdfdf2 Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 25 Sep 2020 10:39:45 -0500 Subject: [PATCH 11/39] logs --- sheepdog/auth/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index e2908d00f..dcdc35e30 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -144,8 +144,8 @@ def create_resource(program, project=None, data=None): person_node = flask.current_app.subject_entity logger.warn(stop_node) logger.warn(person_node) - logger.warn(stop_node[:-1]) - logger.warn(person_node[:-1]) + logger.warn(stop_node.label) + logger.warn(person_node.label) # [:-1] if data and data["type"] == "person": resource += "/persons/{}".format(data["submitter_id"]) From ba94134c9bc78f409195e5ee695385e6f35b6d3b Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 25 Sep 2020 11:11:27 -0500 Subject: [PATCH 12/39] add fix remove logs --- sheepdog/auth/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index dcdc35e30..9fd6fdfb5 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -140,17 +140,18 @@ def create_resource(program, project=None, data=None): if project: resource += "/projects/{}".format(project) + stop_node = flask.current_app.node_authz_entity person_node = flask.current_app.subject_entity - logger.warn(stop_node) - logger.warn(person_node) - logger.warn(stop_node.label) - logger.warn(person_node.label) # [:-1] - - if data and data["type"] == "person": + if data and data["type"] == person_node.label: resource += "/persons/{}".format(data["submitter_id"]) - elif data and data["type"] == "subject": - resource += "/persons/{}/subjects/{}".format(data["persons"][0]["submitter_id"], data["submitter_id"]) + elif data and data["type"] == stop_node.label: + person = None + if isinstance(data["persons"], list): + person = data["persons"][0] + else: + person = data["persons"] + resource += "/persons/{}/subjects/{}".format(person.["submitter_id"], data["submitter_id"]) logger.warn(resource) From b4fc68e619b936496aba91513aa6491a1fa1649e Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 25 Sep 2020 11:27:31 -0500 Subject: [PATCH 13/39] fix --- sheepdog/auth/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index 9fd6fdfb5..fc9f6c641 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -151,7 +151,7 @@ def create_resource(program, project=None, data=None): person = data["persons"][0] else: person = data["persons"] - resource += "/persons/{}/subjects/{}".format(person.["submitter_id"], data["submitter_id"]) + resource += "/persons/{}/subjects/{}".format(person["submitter_id"], data["submitter_id"]) logger.warn(resource) From 847fe0efd6bda7cd6095d9b14777b8183427a91f Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 12 Mar 2021 15:28:57 -0600 Subject: [PATCH 14/39] fix bug in deeper resource auth --- sheepdog/auth/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index fc9f6c641..d897c16f5 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -181,7 +181,7 @@ def check_resource_access(program, project, nodes): subject_submitter_ids.append({"id": node.node_id, "submitter_id": node.props.get("submitter_id", None)}) else: for link in node._pg_links: - tmp_dad = getattr(node, link) + tmp_dad = getattr(node, link)[0] nodeType = link path_tmp = nodeType tmp = node._pg_links[link]["dst_type"] @@ -194,7 +194,7 @@ def check_resource_access(program, project, nodes): tmp_dad = getattr(tmp_dad, nodeType)[0] if tmp.label == stop_node: - subject_submitter_ids.append({"id": tmp_dad[0].node_id, "submitter_id": tmp_dad[0].props.get("submitter_id", None)}) + subject_submitter_ids.append({"id": tmp_dad.node_id, "submitter_id": tmp_dad.props.get("submitter_id", None)}) else: logger.warn("resource not found " + node.label) logger.warn(node) From ac1692a81444ee6857d4515a8fc1b6284ce2b319 Mon Sep 17 00:00:00 2001 From: grugna Date: Tue, 6 Apr 2021 10:58:57 -0500 Subject: [PATCH 15/39] Update __init__.py --- sheepdog/auth/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index d897c16f5..ad5d9ed99 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -140,7 +140,16 @@ def create_resource(program, project=None, data=None): if project: resource += "/projects/{}".format(project) + if type(data) is list: + for d in data: + get_and_create_resource_values(resource, d) + else: + get_and_create_resource_values(resource, data) + + + +def get_and_create_resource_values(resource, data): stop_node = flask.current_app.node_authz_entity person_node = flask.current_app.subject_entity if data and data["type"] == person_node.label: From 19f53bdde568a7662f13422ef7d757008c0fd426 Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 16 Apr 2021 15:44:53 -0500 Subject: [PATCH 16/39] cleanup --- Dockerfile | 2 -- sheepdog/auth/__init__.py | 14 ++++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index f3a53103a..38995a190 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,6 @@ RUN python -m pip install --upgrade pip \ && python -m pip install --upgrade setuptools \ && pip --version \ && pip install -r requirements.txt - # && pip uninstall authlib -y \ - # && pip install authlib==0.14.2 RUN mkdir -p /var/www/sheepdog \ && mkdir /run/ngnix/ \ diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index ad5d9ed99..256e6b5c7 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -111,13 +111,13 @@ def authorize_and_call(*args, **kwargs): return authorize_and_call -def authorize(program, project, roles, resources_tmp=None): +def authorize(program, project, roles, resource_list=None): resource = "/programs/{}/projects/{}".format(program, project) resources = [] - if resources_tmp: - for resource_tmp in resources_tmp: - resources.append(resource + resource_tmp) + if resource_list: + for res in resource_list: + resources.append(resource + res) else: resources = [resource] @@ -131,10 +131,6 @@ def authorize(program, project, roles, resources_tmp=None): def create_resource(program, project=None, data=None): - logger.warn("LUCA RESOURCE ATTENTION") - logger.warn(data) # {'type': 'subject', 'persons': [{'submitter_id': 'lavefrrg'}], 'submitter_id': 'test_sub_1', 'state': 'validated'} - - resource = "/programs/{}".format(program) if project: @@ -146,8 +142,6 @@ def create_resource(program, project=None, data=None): else: get_and_create_resource_values(resource, data) - - def get_and_create_resource_values(resource, data): stop_node = flask.current_app.node_authz_entity From e9a2e6484349f5947885bfa020177e09927c2705 Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 6 May 2021 16:15:03 -0500 Subject: [PATCH 17/39] fix syntax --- sheepdog/api.py | 2 +- sheepdog/auth/__init__.py | 13 ++++++------- sheepdog/blueprint/routes/views/program/project.py | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/sheepdog/api.py b/sheepdog/api.py index 815394fab..6bf268d85 100644 --- a/sheepdog/api.py +++ b/sheepdog/api.py @@ -174,7 +174,7 @@ def app_init(app): if i.lower() == app.node_authz_entity_name.lower(): attribute = getattr(mymodule, i) app.node_authz_entity = attribute - + app = Flask(__name__) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index 256e6b5c7..8fb783a16 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -136,7 +136,7 @@ def create_resource(program, project=None, data=None): if project: resource += "/projects/{}".format(project) - if type(data) is list: + if isinstance(data, list): for d in data: get_and_create_resource_values(resource, d) else: @@ -183,22 +183,22 @@ def check_resource_access(program, project, nodes): if node.label == stop_node: subject_submitter_ids.append({"id": node.node_id, "submitter_id": node.props.get("submitter_id", None)}) else: - for link in node._pg_links: + for link in node._pg_links: tmp_dad = getattr(node, link)[0] nodeType = link path_tmp = nodeType - tmp = node._pg_links[link]["dst_type"] + tmp = node._pg_links[link]["dst_type"] while tmp.label != stop_node and tmp.label != "program": # assuming ony one parents nodeType = list(tmp._pg_links.keys())[0] - path_tmp = path_tmp + "." + nodeType + path_tmp = path_tmp + "." + nodeType tmp = tmp._pg_links[nodeType]["dst_type"] # TODO double check this with deeper relationship > 2 nodes under project tmp_dad = getattr(tmp_dad, nodeType)[0] if tmp.label == stop_node: subject_submitter_ids.append({"id": tmp_dad.node_id, "submitter_id": tmp_dad.props.get("submitter_id", None)}) - else: + else: logger.warn("resource not found " + node.label) logger.warn(node) @@ -225,7 +225,7 @@ def get_authorized_ids(program, project): base_resource_path = "/programs/{}/projects/{}".format(program, project) result = [resource_path for resource_path, permissions in mapping.items() if base_resource_path in resource_path] ids = [] - + for path in result: parts = path.strip("/").split("/") if path != "/" and parts[0] != "programs": @@ -236,7 +236,6 @@ def get_authorized_ids(program, project): if len(parts) < 6: return(None) - break else: ids.append(parts[5]) diff --git a/sheepdog/blueprint/routes/views/program/project.py b/sheepdog/blueprint/routes/views/program/project.py index 327d1a25e..f5418baae 100644 --- a/sheepdog/blueprint/routes/views/program/project.py +++ b/sheepdog/blueprint/routes/views/program/project.py @@ -285,7 +285,7 @@ def get_entities_by_id(program, project, entity_id_string): if missing_entities: raise UserError( "Not found: {}".format(", ".join(missing_entities), code=404) - ) + ) return flask.jsonify({"entities": utils.create_entity_list(entities.values())}) From 919047423c524bc4224513281993139898311dc2 Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 6 May 2021 16:45:50 -0500 Subject: [PATCH 18/39] Update api.py --- tests/integration/datadict/api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/datadict/api.py b/tests/integration/datadict/api.py index 121a93dee..6eaca8232 100644 --- a/tests/integration/datadict/api.py +++ b/tests/integration/datadict/api.py @@ -83,6 +83,21 @@ def app_init(app): except AssertionError: app.logger.info("Blueprint is already registered!!!") + app.node_authz_entity_name = os.environ.get("AUTHZ_ENTITY_NAME", None) + app.node_authz_entity = None + app.subject_entity = None + if app.node_authz_entity_name: + full_module_name = "datamodelutils.models" + mymodule = importlib.import_module(full_module_name) + for i in dir(mymodule): + app.logger.warn(i) + if i.lower() == "person": + attribute = getattr(mymodule, i) + app.subject_entity = attribute + if i.lower() == app.node_authz_entity_name.lower(): + attribute = getattr(mymodule, i) + app.node_authz_entity = attribute + app = Flask(__name__) From d700e997f0e6267906d74fda7b69028536537c67 Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 6 May 2021 16:47:45 -0500 Subject: [PATCH 19/39] Update api.py --- tests/integration/datadictwithobjid/api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/datadictwithobjid/api.py b/tests/integration/datadictwithobjid/api.py index f5467760a..381e2659f 100644 --- a/tests/integration/datadictwithobjid/api.py +++ b/tests/integration/datadictwithobjid/api.py @@ -83,6 +83,21 @@ def app_init(app): except AssertionError: app.logger.info("Blueprint is already registered!!!") + app.node_authz_entity_name = os.environ.get("AUTHZ_ENTITY_NAME", None) + app.node_authz_entity = None + app.subject_entity = None + if app.node_authz_entity_name: + full_module_name = "datamodelutils.models" + mymodule = importlib.import_module(full_module_name) + for i in dir(mymodule): + app.logger.warn(i) + if i.lower() == "person": + attribute = getattr(mymodule, i) + app.subject_entity = attribute + if i.lower() == app.node_authz_entity_name.lower(): + attribute = getattr(mymodule, i) + app.node_authz_entity = attribute + app = Flask(__name__) From 1f060c880797635cc7226aa037640ef8ecce8d3e Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 6 May 2021 21:22:12 -0500 Subject: [PATCH 20/39] add missing import --- tests/integration/datadict/api.py | 2 ++ tests/integration/datadictwithobjid/api.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/integration/datadict/api.py b/tests/integration/datadict/api.py index 6eaca8232..d0803ff5a 100644 --- a/tests/integration/datadict/api.py +++ b/tests/integration/datadict/api.py @@ -7,6 +7,8 @@ import sqlite3 import sys +import os +import importlib import cdis_oauth2client from cdis_oauth2client import OAuth2Client, OAuth2Error diff --git a/tests/integration/datadictwithobjid/api.py b/tests/integration/datadictwithobjid/api.py index 381e2659f..60d489cc5 100644 --- a/tests/integration/datadictwithobjid/api.py +++ b/tests/integration/datadictwithobjid/api.py @@ -7,6 +7,8 @@ import sqlite3 import sys +import os +import importlib import cdis_oauth2client from cdis_oauth2client import OAuth2Client, OAuth2Error From 7155d8155eed61b1a9f5024795eb2a8163cd3af1 Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 6 May 2021 22:31:24 -0500 Subject: [PATCH 21/39] Update api.py --- tests/integration/datadict/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/datadict/api.py b/tests/integration/datadict/api.py index d0803ff5a..382d07f27 100644 --- a/tests/integration/datadict/api.py +++ b/tests/integration/datadict/api.py @@ -20,6 +20,7 @@ from indexd.alias.drivers.alchemy import SQLAlchemyAliasDriver from indexd.auth.drivers.alchemy import SQLAlchemyAuthDriver from psqlgraph import PsqlGraphDriver +from datamodelutils import models, validators, postgres_admin import sheepdog from sheepdog.errors import APIError, setup_default_handlers, UnhealthyCheck From ed8111df2554f8ecececb063ab574cfe823ed866 Mon Sep 17 00:00:00 2001 From: Luca Graglia Date: Thu, 6 Jan 2022 09:53:54 -0600 Subject: [PATCH 22/39] Update and rename LICENSE to LICENSE.md --- LICENSE | 201 ----------------------------------------------------- LICENSE.md | 2 + 2 files changed, 2 insertions(+), 201 deletions(-) delete mode 100644 LICENSE create mode 100644 LICENSE.md diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 261eeb9e9..000000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 000000000..67de100f5 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,2 @@ +[![Creative Commons License](https://i.creativecommons.org/l/by-nc/4.0/88x31.png)](http://creativecommons.org/licenses/by-nc/4.0/) +This work is licensed under a [Creative Commons Attribution-NonCommercial 4.0 International License](http://creativecommons.org/licenses/by-nc/4.0/). From 720dd5447fd80818f8a470f1c8bb45cc15a21368 Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 27 Jan 2022 16:50:30 -0600 Subject: [PATCH 23/39] patch to self referenced outerjoin This is an hardcoded patch to quickly solve the issue. It will need further work to generalize. --- sheepdog/utils/transforms/graph_to_doc.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 51d0598e4..38094aa4c 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -15,6 +15,8 @@ import flask import psqlgraph +from sqlalchemy.orm import aliased + from sheepdog import auth from sheepdog import dictionary from sheepdog.errors import ( @@ -829,11 +831,31 @@ def export_all(node_label, project_id, file_format, db, without_id): # Join the related node tables using the links. for link in cls._pg_links.values(): - query = ( - query.outerjoin(link["edge_out"]) - .outerjoin(link["dst_type"]) - .order_by("src_id") - ) + if link["edge_out"] != "_TimingPartOfTiming_out": + query = ( + query.outerjoin(link["edge_out"]) + .outerjoin(link["dst_type"]) + .order_by("src_id") + ) + else: + # edges = psqlgraph.Edge.get_subclasses() + # edge = psqlgraph.Edge.get_subclass("timingpartoftiming") + # edge = psqlgraph.Edge.get_subclass(link["edge_out"]) + edge = get_unique_subclass("timing", "part_of", "timing") + # print(edge) + + node_timing_dst = aliased(link["dst_type"]) + # userSkillI = aliased(UserSkill) + + # join(userSkillF, User.skills).\ + # join(userSkillI, User.skills).\ + + query = ( + query.outerjoin(edge, cls.node_id==edge.src_id) + .outerjoin(node_timing_dst, node_timing_dst.node_id==edge.dst_id) + .order_by("src_id") + ) + # The result from the query should look like this (header just for # example): # From 006c00e5a363881f21938eee56cd8053b33f3db4 Mon Sep 17 00:00:00 2001 From: grugna Date: Thu, 27 Jan 2022 19:06:54 -0600 Subject: [PATCH 24/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 38094aa4c..6f6121801 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -841,7 +841,7 @@ def export_all(node_label, project_id, file_format, db, without_id): # edges = psqlgraph.Edge.get_subclasses() # edge = psqlgraph.Edge.get_subclass("timingpartoftiming") # edge = psqlgraph.Edge.get_subclass(link["edge_out"]) - edge = get_unique_subclass("timing", "part_of", "timing") + edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") # print(edge) node_timing_dst = aliased(link["dst_type"]) From 4f2666056b34c3e1917234cd99c4689512d748dd Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 22 Apr 2022 13:32:23 -0500 Subject: [PATCH 25/39] Update __init__.py --- sheepdog/auth/__init__.py | 41 ++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index 8fb783a16..e4af0722d 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -184,23 +184,28 @@ def check_resource_access(program, project, nodes): subject_submitter_ids.append({"id": node.node_id, "submitter_id": node.props.get("submitter_id", None)}) else: for link in node._pg_links: - tmp_dad = getattr(node, link)[0] - nodeType = link - path_tmp = nodeType - tmp = node._pg_links[link]["dst_type"] - while tmp.label != stop_node and tmp.label != "program": - # assuming ony one parents - nodeType = list(tmp._pg_links.keys())[0] - path_tmp = path_tmp + "." + nodeType - tmp = tmp._pg_links[nodeType]["dst_type"] - # TODO double check this with deeper relationship > 2 nodes under project - tmp_dad = getattr(tmp_dad, nodeType)[0] - - if tmp.label == stop_node: - subject_submitter_ids.append({"id": tmp_dad.node_id, "submitter_id": tmp_dad.props.get("submitter_id", None)}) - else: - logger.warn("resource not found " + node.label) - logger.warn(node) + print(link) + print(node) + tmp_dads = getattr(node, link, None) + print(tmp_dads) + if tmp_dads: + tmp_dad = tmp_dads[0] + nodeType = link + path_tmp = nodeType + tmp = node._pg_links[link]["dst_type"] + while tmp.label != stop_node and tmp.label != "program": + # assuming ony one parents + nodeType = list(tmp._pg_links.keys())[0] + path_tmp = path_tmp + "." + nodeType + tmp = tmp._pg_links[nodeType]["dst_type"] + # TODO double check this with deeper relationship > 2 nodes under project + tmp_dad = getattr(tmp_dad, nodeType)[0] + + if tmp.label == stop_node: + subject_submitter_ids.append({"id": tmp_dad.node_id, "submitter_id": tmp_dad.props.get("submitter_id", None)}) + else: + logger.warn("resource not found " + node.label) + logger.warn(node) try: resources = [ @@ -216,7 +221,7 @@ def check_resource_access(program, project, nodes): def get_authorized_ids(program, project): try: mapping = flask.current_app.auth.auth_mapping(current_user.username) - except ArboristError as e: + except AuthZError as e: logger.warn( "Unable to retrieve auth mapping for user `{}`: {}".format(current_user.username, e) ) From b29ad04d09f6a8585c3c8e92d9e5b22baa64625d Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 22 Apr 2022 14:27:55 -0500 Subject: [PATCH 26/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 6f6121801..6681412b1 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -802,6 +802,9 @@ def export_all(node_label, project_id, file_format, db, without_id): # class). titles_non_linked, titles_linked = get_all_titles(node_label, without_id) + print("MARCOOOOOOO 1") + print(titles_non_linked) + print(titles_linked) with db.session_scope() as session: # ``linked_props`` is a list of attributes belonging to linked classes # (for example, ``Experiment.node_id``). @@ -818,8 +821,9 @@ def export_all(node_label, project_id, file_format, db, without_id): # Now, fill out the properties lists from the titles. cls = psqlgraph.Node.get_subclass(node_label) linked_props = make_linked_props(cls, titles_linked) + print("MARCOOOOOOO 2") - # Bui ld up the query. The query will contain, firstly, the node class, + # Build up the query. The query will contain, firstly, the node class, # and secondly, all the relevant properties in linked nodes. query_args = [cls] + linked_props query = session.query(*query_args).prop("project_id", project_id) @@ -829,15 +833,22 @@ def export_all(node_label, project_id, file_format, db, without_id): if auth_ids: query = query.prop_in('submitter_id', auth_ids) + print("MARCOOOOOOO 3") + print(query) + print(cls._pg_links.values()) + # Join the related node tables using the links. for link in cls._pg_links.values(): + print(link) if link["edge_out"] != "_TimingPartOfTiming_out": + print("INSIDE MARCOOOOOOO") query = ( query.outerjoin(link["edge_out"]) .outerjoin(link["dst_type"]) .order_by("src_id") ) else: + print("OUTSIDE MARCOOOOOOO") # edges = psqlgraph.Edge.get_subclasses() # edge = psqlgraph.Edge.get_subclass("timingpartoftiming") # edge = psqlgraph.Edge.get_subclass(link["edge_out"]) From e792a8032488859722a8271dd4578e27c4c1445f Mon Sep 17 00:00:00 2001 From: grugna Date: Fri, 22 Apr 2022 15:01:21 -0500 Subject: [PATCH 27/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 6681412b1..ec71a2197 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -802,9 +802,6 @@ def export_all(node_label, project_id, file_format, db, without_id): # class). titles_non_linked, titles_linked = get_all_titles(node_label, without_id) - print("MARCOOOOOOO 1") - print(titles_non_linked) - print(titles_linked) with db.session_scope() as session: # ``linked_props`` is a list of attributes belonging to linked classes # (for example, ``Experiment.node_id``). @@ -821,7 +818,6 @@ def export_all(node_label, project_id, file_format, db, without_id): # Now, fill out the properties lists from the titles. cls = psqlgraph.Node.get_subclass(node_label) linked_props = make_linked_props(cls, titles_linked) - print("MARCOOOOOOO 2") # Build up the query. The query will contain, firstly, the node class, # and secondly, all the relevant properties in linked nodes. @@ -834,8 +830,6 @@ def export_all(node_label, project_id, file_format, db, without_id): query = query.prop_in('submitter_id', auth_ids) print("MARCOOOOOOO 3") - print(query) - print(cls._pg_links.values()) # Join the related node tables using the links. for link in cls._pg_links.values(): @@ -853,7 +847,8 @@ def export_all(node_label, project_id, file_format, db, without_id): # edge = psqlgraph.Edge.get_subclass("timingpartoftiming") # edge = psqlgraph.Edge.get_subclass(link["edge_out"]) edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") - # print(edge) + print(edge) + node_timing_dst = aliased(link["dst_type"]) # userSkillI = aliased(UserSkill) @@ -867,6 +862,9 @@ def export_all(node_label, project_id, file_format, db, without_id): .order_by("src_id") ) + print("MARCOOOOOOO 4") + print(query) + # The result from the query should look like this (header just for # example): # From 0665969ae3839560c85a6f290a29b538110a2d58 Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 10:00:39 -0500 Subject: [PATCH 28/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index ec71a2197..80ba4c140 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -829,26 +829,21 @@ def export_all(node_label, project_id, file_format, db, without_id): if auth_ids: query = query.prop_in('submitter_id', auth_ids) - print("MARCOOOOOOO 3") - # Join the related node tables using the links. for link in cls._pg_links.values(): print(link) if link["edge_out"] != "_TimingPartOfTiming_out": - print("INSIDE MARCOOOOOOO") query = ( query.outerjoin(link["edge_out"]) .outerjoin(link["dst_type"]) .order_by("src_id") ) else: - print("OUTSIDE MARCOOOOOOO") # edges = psqlgraph.Edge.get_subclasses() # edge = psqlgraph.Edge.get_subclass("timingpartoftiming") # edge = psqlgraph.Edge.get_subclass(link["edge_out"]) edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") - print(edge) - + # print(edge) node_timing_dst = aliased(link["dst_type"]) # userSkillI = aliased(UserSkill) @@ -862,8 +857,6 @@ def export_all(node_label, project_id, file_format, db, without_id): .order_by("src_id") ) - print("MARCOOOOOOO 4") - print(query) # The result from the query should look like this (header just for # example): @@ -884,7 +877,10 @@ def export_all(node_label, project_id, file_format, db, without_id): js_list_separator = "" last_id = None current_obj = None - for result in query.yield_per(1000): + partial_results = query.yield_per(1000) + print("MARCOOOOOO 2000") + print(partial_results[0]) + for result in partial_results: node = result[0] node_id = node["node_id"] if node_id != last_id: @@ -892,6 +888,8 @@ def export_all(node_label, project_id, file_format, db, without_id): prop: list_to_comma_string(node[prop], file_format) for prop in props } + print("MARCOOOOOO 203") + print(new_obj) if current_obj != None: yield from yield_result( current_obj, @@ -904,6 +902,8 @@ def export_all(node_label, project_id, file_format, db, without_id): last_id = node_id current_obj = new_obj current_obj = append_links_to_obj(result, current_obj, titles_linked) + print("MARCOOOOOO 205") + print(current_obj) if current_obj is not None: yield from yield_result( @@ -967,6 +967,11 @@ def result_to_delimited_file(props_values, file_format): def append_links_to_obj(result, current_obj, titles_linked): link_props_split = list(map(format_linked_prop, titles_linked)) linked_fields = defaultdict(defaultdict) + + print("MARCOOOOOO link") + print(link_props_split) + print(linked_fields) + print(result) for idx, (link_name, link_prop) in enumerate(link_props_split): if result[idx + 1] is None: continue From d0722866c41c27b20fe85e775512268a3f0b0cc7 Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 10:32:49 -0500 Subject: [PATCH 29/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 80ba4c140..37fbf084d 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -874,6 +874,8 @@ def export_all(node_label, project_id, file_format, db, without_id): # Yield the lines of the file. yield "{}\n".format("\t".join(titles_non_linked + titles_linked)) + print("MARCOOOOOO 4555") + print(query) js_list_separator = "" last_id = None current_obj = None @@ -970,7 +972,7 @@ def append_links_to_obj(result, current_obj, titles_linked): print("MARCOOOOOO link") print(link_props_split) - print(linked_fields) + print(linked_fields.items()) print(result) for idx, (link_name, link_prop) in enumerate(link_props_split): if result[idx + 1] is None: From f79b0e887f8661b0884837df7f28c89b66ffc2d9 Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 13:06:55 -0500 Subject: [PATCH 30/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 37fbf084d..5d70795bd 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -821,6 +821,8 @@ def export_all(node_label, project_id, file_format, db, without_id): # Build up the query. The query will contain, firstly, the node class, # and secondly, all the relevant properties in linked nodes. + print("MARCOOOOOO linked") + print(linked_props) query_args = [cls] + linked_props query = session.query(*query_args).prop("project_id", project_id) From ce04af9e2fc4c0ca773777a6efc90dc2cd6c0291 Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 13:23:15 -0500 Subject: [PATCH 31/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 5d70795bd..73b62dd61 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -823,6 +823,20 @@ def export_all(node_label, project_id, file_format, db, without_id): # and secondly, all the relevant properties in linked nodes. print("MARCOOOOOO linked") print(linked_props) + print(titles_linked) + + + + # # Create alias if joining on itself + # edge = None + # node_timing_dst = None + # for link in cls._pg_links.values(): + # if link["edge_out"] == "_TimingPartOfTiming_out": + # edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") + # node_timing_dst = aliased(link["dst_type"]) + + + query_args = [cls] + linked_props query = session.query(*query_args).prop("project_id", project_id) From c23a670cba5315ea1ac4777f88b60e0ba2634724 Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 13:47:36 -0500 Subject: [PATCH 32/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 36 ++++++++++------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 73b62dd61..1047d1fdd 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -817,27 +817,24 @@ def export_all(node_label, project_id, file_format, db, without_id): # This is used to look up the classes for the linked nodes. # Now, fill out the properties lists from the titles. cls = psqlgraph.Node.get_subclass(node_label) - linked_props = make_linked_props(cls, titles_linked) - - # Build up the query. The query will contain, firstly, the node class, - # and secondly, all the relevant properties in linked nodes. - print("MARCOOOOOO linked") - print(linked_props) - print(titles_linked) - + # Create alias if joining on itself + recursive_node = '_TimingPartOfTiming_out' + edge = None + node_timing_dst = None + for link in cls._pg_links.values(): + if link["edge_out"] == recursive_node: + titles_linked = [a for a in titles_linked if 'timings.' not in a] - # # Create alias if joining on itself - # edge = None - # node_timing_dst = None - # for link in cls._pg_links.values(): - # if link["edge_out"] == "_TimingPartOfTiming_out": - # edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") - # node_timing_dst = aliased(link["dst_type"]) + edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") + node_timing_dst = aliased(link["dst_type"]) + linked_props = make_linked_props(cls, titles_linked) - query_args = [cls] + linked_props + # Build up the query. The query will contain, firstly, the node class, + # and secondly, all the relevant properties in linked nodes. + query_args = [cls] + linked_props + ([node_timing_dst.id, node_timing_dst.submitter_id] if node_timing_dst is not None else []) query = session.query(*query_args).prop("project_id", project_id) #add filter by id the user is authorized to access @@ -847,8 +844,7 @@ def export_all(node_label, project_id, file_format, db, without_id): # Join the related node tables using the links. for link in cls._pg_links.values(): - print(link) - if link["edge_out"] != "_TimingPartOfTiming_out": + if link["edge_out"] != recursive_node: query = ( query.outerjoin(link["edge_out"]) .outerjoin(link["dst_type"]) @@ -858,10 +854,10 @@ def export_all(node_label, project_id, file_format, db, without_id): # edges = psqlgraph.Edge.get_subclasses() # edge = psqlgraph.Edge.get_subclass("timingpartoftiming") # edge = psqlgraph.Edge.get_subclass(link["edge_out"]) - edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") + # edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") # print(edge) - node_timing_dst = aliased(link["dst_type"]) + # node_timing_dst = aliased(link["dst_type"]) # userSkillI = aliased(UserSkill) # join(userSkillF, User.skills).\ From 29bbecda2bcd2c574d8f0841e699d3f116eef94e Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 14:14:04 -0500 Subject: [PATCH 33/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 1047d1fdd..0912d2c79 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -834,7 +834,11 @@ def export_all(node_label, project_id, file_format, db, without_id): # Build up the query. The query will contain, firstly, the node class, # and secondly, all the relevant properties in linked nodes. - query_args = [cls] + linked_props + ([node_timing_dst.id, node_timing_dst.submitter_id] if node_timing_dst is not None else []) + query_args = [cls] + linked_props + print("MARCOOOOOO 5555") + print(query_args) + query_args.extend([node_timing_dst.id, node_timing_dst.submitter_id] if node_timing_dst is not None else []) + print(query_args) query = session.query(*query_args).prop("project_id", project_id) #add filter by id the user is authorized to access From 799bd40b71035ff8b5915cd33f51f8199bdc837c Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 15:14:34 -0500 Subject: [PATCH 34/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 0912d2c79..c1c70dc35 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -827,17 +827,30 @@ def export_all(node_label, project_id, file_format, db, without_id): titles_linked = [a for a in titles_linked if 'timings.' not in a] edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") - node_timing_dst = aliased(link["dst_type"]) + # node_timing_dst = aliased(link["dst_type"]) + node_timing_dst = aliased(link["dst_type"], name='node_timing_1') + + print("MARCOOOOOO 5555556") + print(titles_linked) + print(cls) + print(map(format_linked_prop, titles_linked)) + print(list(map(format_linked_prop, titles_linked))) + print([ getattr(cls._pg_links[link_name]["dst_type"], link_prop) for (link_name, link_prop) in list(map(format_linked_prop, titles_linked))]) + print("END") + linked_props = make_linked_props(cls, titles_linked) + + # Build up the query. The query will contain, firstly, the node class, # and secondly, all the relevant properties in linked nodes. query_args = [cls] + linked_props print("MARCOOOOOO 5555") + # ['subjects.id', 'subjects.submitter_id', 'timings.id', 'timings.submitter_id'] print(query_args) - query_args.extend([node_timing_dst.id, node_timing_dst.submitter_id] if node_timing_dst is not None else []) + # query_args.extend([node_timing_dst.id, node_timing_dst.submitter_id] if node_timing_dst is not None else []) print(query_args) query = session.query(*query_args).prop("project_id", project_id) From 4c7ab2d5221a25c0c8db9570465fa2e44a082cec Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 15:44:51 -0500 Subject: [PATCH 35/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index c1c70dc35..e5a3f910b 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -850,7 +850,7 @@ def export_all(node_label, project_id, file_format, db, without_id): print("MARCOOOOOO 5555") # ['subjects.id', 'subjects.submitter_id', 'timings.id', 'timings.submitter_id'] print(query_args) - # query_args.extend([node_timing_dst.id, node_timing_dst.submitter_id] if node_timing_dst is not None else []) + query_args.extend([getattr(node_timing_dst, 'id'), getattr(node_timing_dst, 'submitter_id')] if node_timing_dst is not None else []) print(query_args) query = session.query(*query_args).prop("project_id", project_id) From e377fd0dcb605b8ce936349b8d5a824ee23b928f Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 15:54:55 -0500 Subject: [PATCH 36/39] Update graph_to_doc.py --- sheepdog/utils/transforms/graph_to_doc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index e5a3f910b..0ffbdb59b 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -836,6 +836,9 @@ def export_all(node_label, project_id, file_format, db, without_id): print(map(format_linked_prop, titles_linked)) print(list(map(format_linked_prop, titles_linked))) print([ getattr(cls._pg_links[link_name]["dst_type"], link_prop) for (link_name, link_prop) in list(map(format_linked_prop, titles_linked))]) + print(getattr(node_timing_dst, 'id')) + print(getattr(node_timing_dst, 'node_id')) + print(getattr(node_timing_dst, 'submitter_id')) print("END") @@ -850,7 +853,7 @@ def export_all(node_label, project_id, file_format, db, without_id): print("MARCOOOOOO 5555") # ['subjects.id', 'subjects.submitter_id', 'timings.id', 'timings.submitter_id'] print(query_args) - query_args.extend([getattr(node_timing_dst, 'id'), getattr(node_timing_dst, 'submitter_id')] if node_timing_dst is not None else []) + query_args.extend([getattr(node_timing_dst, 'node_id'), getattr(node_timing_dst, 'submitter_id')] if node_timing_dst is not None else []) print(query_args) query = session.query(*query_args).prop("project_id", project_id) From 95a4314d59e88a236da46f40f745cb089ce0942c Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 16:09:09 -0500 Subject: [PATCH 37/39] clean up debugging prints --- sheepdog/utils/transforms/graph_to_doc.py | 52 ++++------------------- 1 file changed, 9 insertions(+), 43 deletions(-) diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index 0ffbdb59b..f7515c918 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -825,36 +825,26 @@ def export_all(node_label, project_id, file_format, db, without_id): for link in cls._pg_links.values(): if link["edge_out"] == recursive_node: titles_linked = [a for a in titles_linked if 'timings.' not in a] - edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") # node_timing_dst = aliased(link["dst_type"]) + # edges = psqlgraph.Edge.get_subclasses() + # edge = psqlgraph.Edge.get_subclass("timingpartoftiming") + # edge = psqlgraph.Edge.get_subclass(link["edge_out"]) + # edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") + # print(edge) + # node_timing_dst = aliased(link["dst_type"]) + # userSkillI = aliased(UserSkill) + # join(userSkillF, User.skills).\ + # join(userSkillI, User.skills).\ node_timing_dst = aliased(link["dst_type"], name='node_timing_1') - print("MARCOOOOOO 5555556") - print(titles_linked) - print(cls) - print(map(format_linked_prop, titles_linked)) - print(list(map(format_linked_prop, titles_linked))) - print([ getattr(cls._pg_links[link_name]["dst_type"], link_prop) for (link_name, link_prop) in list(map(format_linked_prop, titles_linked))]) - print(getattr(node_timing_dst, 'id')) - print(getattr(node_timing_dst, 'node_id')) - print(getattr(node_timing_dst, 'submitter_id')) - print("END") - - - linked_props = make_linked_props(cls, titles_linked) - # Build up the query. The query will contain, firstly, the node class, # and secondly, all the relevant properties in linked nodes. query_args = [cls] + linked_props - print("MARCOOOOOO 5555") - # ['subjects.id', 'subjects.submitter_id', 'timings.id', 'timings.submitter_id'] - print(query_args) query_args.extend([getattr(node_timing_dst, 'node_id'), getattr(node_timing_dst, 'submitter_id')] if node_timing_dst is not None else []) - print(query_args) query = session.query(*query_args).prop("project_id", project_id) #add filter by id the user is authorized to access @@ -871,18 +861,6 @@ def export_all(node_label, project_id, file_format, db, without_id): .order_by("src_id") ) else: - # edges = psqlgraph.Edge.get_subclasses() - # edge = psqlgraph.Edge.get_subclass("timingpartoftiming") - # edge = psqlgraph.Edge.get_subclass(link["edge_out"]) - # edge = psqlgraph.Edge.get_unique_subclass("timing", "part_of", "timing") - # print(edge) - - # node_timing_dst = aliased(link["dst_type"]) - # userSkillI = aliased(UserSkill) - - # join(userSkillF, User.skills).\ - # join(userSkillI, User.skills).\ - query = ( query.outerjoin(edge, cls.node_id==edge.src_id) .outerjoin(node_timing_dst, node_timing_dst.node_id==edge.dst_id) @@ -906,14 +884,10 @@ def export_all(node_label, project_id, file_format, db, without_id): # Yield the lines of the file. yield "{}\n".format("\t".join(titles_non_linked + titles_linked)) - print("MARCOOOOOO 4555") - print(query) js_list_separator = "" last_id = None current_obj = None partial_results = query.yield_per(1000) - print("MARCOOOOOO 2000") - print(partial_results[0]) for result in partial_results: node = result[0] node_id = node["node_id"] @@ -922,8 +896,6 @@ def export_all(node_label, project_id, file_format, db, without_id): prop: list_to_comma_string(node[prop], file_format) for prop in props } - print("MARCOOOOOO 203") - print(new_obj) if current_obj != None: yield from yield_result( current_obj, @@ -936,8 +908,6 @@ def export_all(node_label, project_id, file_format, db, without_id): last_id = node_id current_obj = new_obj current_obj = append_links_to_obj(result, current_obj, titles_linked) - print("MARCOOOOOO 205") - print(current_obj) if current_obj is not None: yield from yield_result( @@ -1002,10 +972,6 @@ def append_links_to_obj(result, current_obj, titles_linked): link_props_split = list(map(format_linked_prop, titles_linked)) linked_fields = defaultdict(defaultdict) - print("MARCOOOOOO link") - print(link_props_split) - print(linked_fields.items()) - print(result) for idx, (link_name, link_prop) in enumerate(link_props_split): if result[idx + 1] is None: continue From 630e2f4132ab791e857cd09fd28f253a70e41dd4 Mon Sep 17 00:00:00 2001 From: grugna Date: Mon, 25 Apr 2022 16:48:13 -0500 Subject: [PATCH 38/39] cleanup --- sheepdog/auth/__init__.py | 3 --- sheepdog/utils/transforms/graph_to_doc.py | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/sheepdog/auth/__init__.py b/sheepdog/auth/__init__.py index e4af0722d..bd81faf70 100644 --- a/sheepdog/auth/__init__.py +++ b/sheepdog/auth/__init__.py @@ -184,10 +184,7 @@ def check_resource_access(program, project, nodes): subject_submitter_ids.append({"id": node.node_id, "submitter_id": node.props.get("submitter_id", None)}) else: for link in node._pg_links: - print(link) - print(node) tmp_dads = getattr(node, link, None) - print(tmp_dads) if tmp_dads: tmp_dad = tmp_dads[0] nodeType = link diff --git a/sheepdog/utils/transforms/graph_to_doc.py b/sheepdog/utils/transforms/graph_to_doc.py index f7515c918..25d193b9a 100644 --- a/sheepdog/utils/transforms/graph_to_doc.py +++ b/sheepdog/utils/transforms/graph_to_doc.py @@ -867,7 +867,6 @@ def export_all(node_label, project_id, file_format, db, without_id): .order_by("src_id") ) - # The result from the query should look like this (header just for # example): # @@ -887,8 +886,7 @@ def export_all(node_label, project_id, file_format, db, without_id): js_list_separator = "" last_id = None current_obj = None - partial_results = query.yield_per(1000) - for result in partial_results: + for result in query.yield_per(1000): node = result[0] node_id = node["node_id"] if node_id != last_id: @@ -971,7 +969,6 @@ def result_to_delimited_file(props_values, file_format): def append_links_to_obj(result, current_obj, titles_linked): link_props_split = list(map(format_linked_prop, titles_linked)) linked_fields = defaultdict(defaultdict) - for idx, (link_name, link_prop) in enumerate(link_props_split): if result[idx + 1] is None: continue From 2ea8b9e98661e8b42fe47252388b7d0506f7f683 Mon Sep 17 00:00:00 2001 From: Luca Graglia Date: Fri, 16 Aug 2024 16:27:13 -0500 Subject: [PATCH 39/39] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3841ed935..d42d1dd18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ ENV appname=sheepdog RUN pip install --upgrade pip poetry RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential libffi-dev musl-dev gcc libxml2-dev libxslt-dev \ - curl bash git vim + curl bash git vim postgresql-client RUN mkdir -p /var/www/$appname \ && mkdir -p /var/www/.cache/Python-Eggs/ \