From 464e74febec97912a49d398f6e1e5dd136018c8a Mon Sep 17 00:00:00 2001 From: Jessica Gadling Date: Tue, 19 Sep 2023 10:46:26 -0700 Subject: [PATCH 1/8] Some seed data --- Makefile | 2 +- bin/seed_moto.sh | 8 ++++++-- entities/api/files.py | 2 +- entities/test_infra/factories.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 3ef5c0d2..afdb5074 100644 --- a/Makefile +++ b/Makefile @@ -24,4 +24,4 @@ clean: $(MAKE) -C entities local-clean $(MAKE) -C workflows local-clean docker compose down - rm .moto_recording + rm -f .moto_recording diff --git a/bin/seed_moto.sh b/bin/seed_moto.sh index e5c9c2d8..66059c0e 100755 --- a/bin/seed_moto.sh +++ b/bin/seed_moto.sh @@ -8,5 +8,9 @@ export AWS_SECRET_ACCESS_KEY=test export AWS_REGION=us-west-2 # Create dev bucket but don't error if it already exists -bucket=local-bucket -$aws s3api head-bucket --bucket $bucket 2>/dev/null || $aws s3 mb s3://$bucket +bucket_1=local-bucket +bucket_2=remote-bucket +$aws s3api head-bucket --bucket $bucket_1 2>/dev/null || $aws s3 mb s3://$bucket_1 +$aws s3api head-bucket --bucket $bucket_2 2>/dev/null || $aws s3 mb s3://$bucket_2 +$aws s3 cp entities/test_infra/fixtures/test1.fastq s3://$bucket_1/anything/back/among/population.wav +$aws s3 cp entities/test_infra/fixtures/test1.fastq s3://$bucket_2/remember/offer/radio/result.webm diff --git a/entities/api/files.py b/entities/api/files.py index 13941968..6c6a3b4e 100644 --- a/entities/api/files.py +++ b/entities/api/files.py @@ -26,7 +26,7 @@ def download_link( ) -> typing.Optional[SignedURL]: if not self.path: # type: ignore return None - key = self.path # type: ignore + key = self.path.lstrip("/") # type: ignore bucket_name = self.namespace # type: ignore url = s3_client.generate_presigned_url( ClientMethod="get_object", Params={"Bucket": bucket_name, "Key": key}, ExpiresIn=expiration diff --git a/entities/test_infra/factories.py b/entities/test_infra/factories.py index 22df3474..8c56b7e9 100644 --- a/entities/test_infra/factories.py +++ b/entities/test_infra/factories.py @@ -49,7 +49,7 @@ class Meta: status = factory.Faker("enum", enum_cls=FileStatus) protocol = fuzzy.FuzzyChoice(["S3", "GCP"]) - namespace = fuzzy.FuzzyChoice(["bucket_1", "bucket_2"]) + namespace = fuzzy.FuzzyChoice(["local-bucket", "remote-bucket"]) # path = factory.LazyAttribute(lambda o: {factory.Faker("file_path", depth=3, extension=o.file_format)}) path = factory.Faker("file_path", depth=3) file_format = fuzzy.FuzzyChoice(["fasta", "fastq", "bam"]) From 14a0648052dedaeba25dcb8bec52e9d125e67369 Mon Sep 17 00:00:00 2001 From: Jessica Gadling Date: Tue, 19 Sep 2023 10:57:27 -0700 Subject: [PATCH 2/8] Include small test fastq --- entities/test_infra/fixtures/test1.fastq | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 entities/test_infra/fixtures/test1.fastq diff --git a/entities/test_infra/fixtures/test1.fastq b/entities/test_infra/fixtures/test1.fastq new file mode 100644 index 00000000..b5acc2f9 --- /dev/null +++ b/entities/test_infra/fixtures/test1.fastq @@ -0,0 +1,41 @@ +@MG148341.1_0__benchmark_lineage_0_463676_12059_12058__s0000000000 +CTGGTCAGGTAGCATAACCATTACATTCATGTGTGTTTGTGATGCATTTAGTACTGGTAAATTCCTGCTAGCATACACCCCCCCTGGAGGCGCCCTACCAGCCAATCGAAAGCAGGCAATGTTGGG ++ +BBBBBFFFFFFFFFFFFFFFBFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF/FFFFBFFF/FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF/FFFF Date: Tue, 19 Sep 2023 17:30:10 -0700 Subject: [PATCH 3/8] Handle validating completed uploads. --- entities/.dockerignore => .dockerignore | 11 ++- bin/init_moto.sh | 4 +- docker-compose.yml | 9 ++- entities/api/files.py | 35 +++++++++ entities/api/main.py | 3 +- entities/docker-compose.yml | 4 +- entities/files/format_handlers.py | 40 ++++++++++ entities/poetry.lock | 98 ++++++++++++++++++++++++- entities/pyproject.toml | 1 + platformics/api/core/gql_loaders.py | 77 +++++-------------- platformics/security/__init__.py | 0 platformics/security/authorization.py | 38 ++++++++++ 12 files changed, 246 insertions(+), 74 deletions(-) rename entities/.dockerignore => .dockerignore (52%) mode change 100644 => 100755 bin/init_moto.sh create mode 100644 entities/files/format_handlers.py create mode 100644 platformics/security/__init__.py create mode 100644 platformics/security/authorization.py diff --git a/entities/.dockerignore b/.dockerignore similarity index 52% rename from entities/.dockerignore rename to .dockerignore index 5b601660..af8fa31f 100644 --- a/entities/.dockerignore +++ b/.dockerignore @@ -9,8 +9,11 @@ **/__pycache__ **/*.egg-info **/.cache/ +**/.vscode/ -.tox -.coverage -.pytest_cache -.mypy_cache +**/.tox +**/.coverage +**/.pytest_cache/ +**/.mypy_cache/ +**/.ruff_cache/ +**/.vscode/ diff --git a/bin/init_moto.sh b/bin/init_moto.sh old mode 100644 new mode 100755 index 68d2fde2..4cecc470 --- a/bin/init_moto.sh +++ b/bin/init_moto.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/bash # Script to initialize moto server; runs inside the motoserver container @@ -6,7 +6,7 @@ moto_server --host 0.0.0.0 --port $MOTO_PORT & # Initialize data once server is ready -sleep 1 && curl -X POST "http://localhost:${MOTO_PORT}/moto-api/recorder/replay-recording" +sleep 1 && curl -X POST "http://motoserver.czidnet:${MOTO_PORT}/moto-api/recorder/replay-recording" # Go back to moto server wait diff --git a/docker-compose.yml b/docker-compose.yml index 2680c5bb..9efa4f42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,16 +5,19 @@ services: # To use it from the CLI: aws --endpoint-url=http://localhost:4000 s3 ls # To reset all services without restarting the container: curl -X POST http://localhost:4000/moto-api/reset motoserver: - image: motoserver/moto:4.2.2 + image: motoserver/moto:4.2.3 ports: - "4000:4000" environment: - MOTO_PORT=4000 - MOTO_ENABLE_RECORDING=True + - S3_IGNORE_SUBDOMAIN_BUCKETNAME=True + - MOTO_S3_CUSTOM_ENDPOINTS=http://motoserver.czidnet:4000 volumes: - .moto_recording:/moto/moto_recording - - ./bin/init_moto.sh:/moto/init_moto.sh - entrypoint: ["bash", "/moto/init_moto.sh"] + - ./bin:/moto/bin + entrypoint: [] + command: ["/moto/bin/init_moto.sh"] networks: default: diff --git a/entities/api/files.py b/entities/api/files.py index 6c6a3b4e..2d6058e2 100644 --- a/entities/api/files.py +++ b/entities/api/files.py @@ -1,12 +1,20 @@ import typing import database.models as db import strawberry +import uuid from fastapi import Depends from mypy_boto3_s3.client import S3Client from platformics.api.core.deps import get_s3_client from platformics.api.core.strawberry_extensions import DependencyExtension from api.strawberry import strawberry_sqlalchemy_mapper +from cerbos.sdk.client import CerbosClient +from cerbos.sdk.model import Principal +from platformics.api.core.deps import get_cerbos_client, get_db_session, require_auth_principal +from sqlalchemy.ext.asyncio import AsyncSession +from platformics.security.authorization import CerbosAction, get_resource_query +from files.format_handlers import get_validator + @strawberry.type class SignedURL: @@ -32,3 +40,30 @@ def download_link( ClientMethod="get_object", Params={"Bucket": bucket_name, "Key": key}, ExpiresIn=expiration ) return SignedURL(url=url, protocol="https", method="get", expiration=expiration) + + +@strawberry.mutation(extensions=[DependencyExtension()]) +async def mark_upload_complete( + file_id: uuid.UUID, + principal: Principal = Depends(require_auth_principal), + cerbos_client: CerbosClient = Depends(get_cerbos_client), + session: AsyncSession = Depends(get_db_session, use_cache=False), + s3_client: S3Client = Depends(get_s3_client), + **kwargs: typing.Any, +) -> db.File: + query = get_resource_query(principal, cerbos_client, CerbosAction.UPDATE, db.File) + query = query.filter(db.File.id == file_id) + file = (await session.execute(query)).scalars().one() + if not file: + raise Exception("NOT FOUND!") # TODO: How do we raise sane errors in our api? + + validator = get_validator(file.file_format) + try: + file_size = validator.validate(s3_client, file.namespace, file.path) + except: # noqa + raise Exception("VALIDATION FAILURE!!") + + file.status = db.FileStatus.SUCCESS + file.size = file_size + + return file diff --git a/entities/api/main.py b/entities/api/main.py index 1040665e..e4345f7d 100644 --- a/entities/api/main.py +++ b/entities/api/main.py @@ -18,7 +18,7 @@ from platformics.database.connect import AsyncDB from strawberry.fastapi import GraphQLRouter from api.strawberry import strawberry_sqlalchemy_mapper -from api.files import File +from api.files import File, mark_upload_complete ###################### # Strawberry-GraphQL # @@ -77,6 +77,7 @@ class Mutation: # file_stuff get_upload_url: Sample = get_base_updater(db.Sample, Sample) # type: ignore + mark_upload_complete: File = mark_upload_complete # -------------------- diff --git a/entities/docker-compose.yml b/entities/docker-compose.yml index 9e143aa3..4174284c 100644 --- a/entities/docker-compose.yml +++ b/entities/docker-compose.yml @@ -45,8 +45,8 @@ services: - DEFAULT_UPLOAD_BUCKET=local-bucket - BOTO_ENDPOINT_URL=http://motoserver.czidnet:4000 - AWS_REGION=us-west-2 - - AWS_ACCESS_KEY_ID=ACCESS_ID - - AWS_SECRET_ACCESS_KEY=ACCESS_KEY + - AWS_ACCESS_KEY_ID=test + - AWS_SECRET_ACCESS_KEY=test # TODO - these are keypairs for testing only! Do not use in prod!! - JWK_PUBLIC_KEY_FILE=/czid-platformics/entities/test_infra/fixtures/public_key.pem - JWK_PRIVATE_KEY_FILE=/czid-platformics/entities/test_infra/fixtures/private_key.pem diff --git a/entities/files/format_handlers.py b/entities/files/format_handlers.py new file mode 100644 index 00000000..40eaef39 --- /dev/null +++ b/entities/files/format_handlers.py @@ -0,0 +1,40 @@ +from abc import ABC, abstractmethod +from mypy_boto3_s3.client import S3Client +from Bio import SeqIO +from io import StringIO +from typing import Protocol + + +class FileFormatHandler(Protocol): + @classmethod + @abstractmethod + def validate(cls, client: S3Client, bucket: str, file_path: str) -> int: + raise NotImplementedError + + @classmethod + @abstractmethod + def convert_to(cls, client: S3Client, bucket: str, file_path: str, format: dict) -> str: + raise NotImplementedError + + +class FastqHandler(FileFormatHandler): + @classmethod + def validate(cls, client: S3Client, bucket: str, file_path: str) -> int: + # Overly simplistic validator for fastq filees checks whether the first 1mb of a file are a valid fastq + data = client.get_object(Bucket=bucket, Key=file_path, Range="bytes=0-1000000")["Body"].read() + records = 0 + for _ in SeqIO.parse(StringIO(str(data)), "fasta"): + records += 1 + assert records > 0 + return client.head_object(Bucket=bucket, Key=file_path)["ContentLength"] + + @classmethod + def convert_to(cls, client: S3Client, bucket: str, file_path: str, format: dict) -> str: + return "" + +def get_validator(format: dict) -> type[FileFormatHandler]: + if format["name"] == "fastq": + return FastqHandler + else: + raise Exception("Unknown file format") + diff --git a/entities/poetry.lock b/entities/poetry.lock index 45c4a992..06b2f4bb 100644 --- a/entities/poetry.lock +++ b/entities/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.0 and should not be changed by hand. [[package]] name = "alembic" @@ -103,6 +103,42 @@ files = [ docs = ["Sphinx (>=5.3.0,<5.4.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] test = ["flake8 (>=5.0,<6.0)", "uvloop (>=0.15.3)"] +[[package]] +name = "biopython" +version = "1.81" +description = "Freely available tools for computational molecular biology." +optional = false +python-versions = ">=3.7" +files = [ + {file = "biopython-1.81-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef7c79b65b0b3f3c7dc59e20a7f8ae5758d8e852cb8b9cace590dc5617e348ba"}, + {file = "biopython-1.81-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ebfbce0d91796c7aef422ee9dffe8827e07e5abaa94545e006f1f20e965c80b"}, + {file = "biopython-1.81-cp310-cp310-win32.whl", hash = "sha256:919a2c583cabf9c96d2ae4e1245a6b0376932fb342aca302a0fc198b71ab3275"}, + {file = "biopython-1.81-cp310-cp310-win_amd64.whl", hash = "sha256:b37c0d24191e5c96ca02415a5188551980c83a0d518bbc4ffe3c9a5d1fe0ee81"}, + {file = "biopython-1.81-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7a168709694e10b338718c18d967edd5b56c237dc88642c22275796007a70000"}, + {file = "biopython-1.81-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51d9c1d1b4b634447535da74a644fae59bc234fbbf9001e2dc6b6fbabb98019"}, + {file = "biopython-1.81-cp311-cp311-win32.whl", hash = "sha256:2f9cfaf16d55ab80d514e7aebe5710dabe4e4ff47ede851031202e33b3249da3"}, + {file = "biopython-1.81-cp311-cp311-win_amd64.whl", hash = "sha256:e41b55edcfd448630e77bf4de66a7235324a8a149621499891da6bd1d5085b9a"}, + {file = "biopython-1.81-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3b36ba1bf6395c09a365c53530c9d71f3617763fa2c1d452b3d8948368c0f1de"}, + {file = "biopython-1.81-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c5c07123ff5f44c9e6b5369df854a38afd3c0c50ef58498a0ae8f7eb799f3e8"}, + {file = "biopython-1.81-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:97cbdbed01b2512471f36c74b91658d1dfbdcbf39bc038f6ce5a41c3e60a8fc6"}, + {file = "biopython-1.81-cp37-cp37m-win32.whl", hash = "sha256:35506e39822c52d11cf09a3951e82375ca1bb9303960b4286acf02c9a6f6c4cc"}, + {file = "biopython-1.81-cp37-cp37m-win_amd64.whl", hash = "sha256:793c42a376cd63f62f8a088ce39b7dc6b5c55e4e9031d887c434de1595bfa4b8"}, + {file = "biopython-1.81-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:11d673698b3d0d6589292ea951fb62cb24ea27d273eca0d08dbbd956690f97f5"}, + {file = "biopython-1.81-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:655df416936662c0c8a06a549cb25e1560e1fea5067d850f34fb714b8a3fae6c"}, + {file = "biopython-1.81-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:762c6c43a8486b5fcd07f136a3217b87d24755618b9ea9da1f17124ff44c2ad6"}, + {file = "biopython-1.81-cp38-cp38-win32.whl", hash = "sha256:ee51bb1cd7decffd24da6b76d5e01b7e2fd818ab85cf0c180226cbb5793a3abd"}, + {file = "biopython-1.81-cp38-cp38-win_amd64.whl", hash = "sha256:ccd729249fd5f586dd4c2a3507c2ea2456825d7e615e97c07c409c850eaf4594"}, + {file = "biopython-1.81-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9ba33244f0eff830beaa7240065bdb5095d96fded6599b76bbb9ddab45cd2bbd"}, + {file = "biopython-1.81-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bb0c690c7368f255ed45236bf0f5464b476b8c083c8f634533921af78278261"}, + {file = "biopython-1.81-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65b93b513ce9dd7b2ce058720eadf42cd03f312db3409356efeb93123d1320aa"}, + {file = "biopython-1.81-cp39-cp39-win32.whl", hash = "sha256:811796f8d222aa3869a50e31e54ce62b69106b47cd8bb06934867c0d843297b5"}, + {file = "biopython-1.81-cp39-cp39-win_amd64.whl", hash = "sha256:b09efcb4733c8770f25eab5fe555a96a08f5ab9e1bc36939e08ebf2ffbf3e0f1"}, + {file = "biopython-1.81.tar.gz", hash = "sha256:2cf38112b6d8415ad39d6a611988cd11fb5f33eb09346666a87263beba9614e0"}, +] + +[package.dependencies] +numpy = "*" + [[package]] name = "black" version = "23.7.0" @@ -989,6 +1025,7 @@ files = [ {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, + {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, @@ -997,6 +1034,7 @@ files = [ {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, + {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, @@ -1026,6 +1064,7 @@ files = [ {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, + {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, @@ -1034,6 +1073,7 @@ files = [ {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, + {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, @@ -1479,6 +1519,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1640,6 +1690,40 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "numpy" +version = "1.25.2" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, + {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, + {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, + {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, + {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, + {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, + {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, + {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, + {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, + {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, + {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, + {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, + {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, + {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, + {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, + {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, + {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, + {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, + {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, + {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, + {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, + {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, + {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, +] + [[package]] name = "packaging" version = "23.1" @@ -2105,6 +2189,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2112,8 +2197,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2130,6 +2222,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2137,6 +2230,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2722,4 +2816,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "f444f64e9fc775e4848da803e862a9f76929c138e73c6a8c1d33738119c04580" +content-hash = "d0a11dfd0392919c8832f53b8e7c2d781cef52332aebfa5b175ab03301c8528e" diff --git a/entities/pyproject.toml b/entities/pyproject.toml index 1360944c..85a98c88 100644 --- a/entities/pyproject.toml +++ b/entities/pyproject.toml @@ -38,6 +38,7 @@ types-requests = "^2.31.0.2" boto3 = "^1.28.43" boto3-stubs = {extras = ["s3"], version = "^1.28.50"} faker-enum = "^0.0.2" +biopython = "^1.81" [build-system] requires = ["poetry-core"] diff --git a/platformics/api/core/gql_loaders.py b/platformics/api/core/gql_loaders.py index 5582ede8..c5dd8875 100644 --- a/platformics/api/core/gql_loaders.py +++ b/platformics/api/core/gql_loaders.py @@ -18,61 +18,21 @@ from sqlalchemy.orm import RelationshipProperty from strawberry.arguments import StrawberryArgument from strawberry.dataloader import DataLoader +from platformics.security.authorization import get_resource_query, CerbosAction -CERBOS_ACTION_VIEW = "view" -CERBOS_ACTION_CREATE = "create" -CERBOS_ACTION_UPDATE = "update" - -E = typing.TypeVar("E", bound=db.Entity) +E = typing.TypeVar("E", bound=db.Base) T = typing.TypeVar("T") -async def get_entities( - model_cls: type[db.Entity], - session: AsyncSession, - cerbos_client: CerbosClient, - principal: Principal, - filters: Optional[list[ColumnExpressionArgument]] = [], - order_by: Optional[list[tuple[ColumnElement[Any], ...]]] = [], -) -> typing.Sequence[db.Entity]: - rd = ResourceDesc(model_cls.__tablename__) - plan = cerbos_client.plan_resources(CERBOS_ACTION_VIEW, principal, rd) - query = get_query( - plan, - model_cls, # type: ignore - { - "request.resource.attr.owner_user_id": model_cls.owner_user_id, - "request.resource.attr.collection_id": model_cls.collection_id, - }, - [], - ) - if filters: - query = query.filter(*filters) # type: ignore - if order_by: - query = query.order_by(*order_by) # type: ignore - result = await session.execute(query) - return result.scalars().all() - - -async def get_files( - model_cls: type[db.File], +async def get_db_rows( + model_cls: type[E], session: AsyncSession, cerbos_client: CerbosClient, principal: Principal, filters: Optional[list[ColumnExpressionArgument]] = [], order_by: Optional[list[tuple[ColumnElement[Any], ...]]] = [], -) -> typing.Sequence[db.File]: - rd = ResourceDesc(model_cls.__tablename__) - plan = cerbos_client.plan_resources(CERBOS_ACTION_VIEW, principal, rd) - query = get_query( - plan, - model_cls, # type: ignore - { - "request.resource.attr.owner_user_id": db.Entity.owner_user_id, - "request.resource.attr.collection_id": db.Entity.collection_id, - }, - [(db.Entity, model_cls.entity_id == db.Entity.id)], # type: ignore - ) +) -> typing.Sequence[E]: + query = get_resource_query(principal, cerbos_client, CerbosAction.VIEW, model_cls) if filters: query = query.filter(*filters) # type: ignore if order_by: @@ -105,10 +65,7 @@ def loader_for(self, relationship: RelationshipProperty) -> DataLoader: except KeyError: related_model = relationship.entity.entity - if related_model == db.File: - load_method = get_files # type: ignore - else: - load_method = get_entities # type: ignore + load_method = get_db_rows # type: ignore async def load_fn(keys: list[Tuple]) -> typing.Sequence[Any]: if not relationship.local_remote_pairs: @@ -169,7 +126,7 @@ async def resolve_entity( filters = [] if id: filters.append(sql_model.id == id) - return await get_entities(sql_model, session, cerbos_client, principal, filters, []) # type: ignore + return await get_db_rows(sql_model, session, cerbos_client, principal, filters, []) # type: ignore return typing.cast(typing.Sequence[T], resolve_entity) @@ -185,7 +142,7 @@ async def resolve_file( filters = [] if id: filters.append(sql_model.id == id) - return await get_files(sql_model, session, cerbos_client, principal, filters, []) # type: ignore + return await get_db_rows(sql_model, session, cerbos_client, principal, filters, []) # type: ignore return typing.cast(typing.Sequence[T], resolve_file) @@ -217,7 +174,7 @@ async def create( return new_entity create.arguments = generate_strawberry_arguments( - CERBOS_ACTION_CREATE, sql_model, gql_type + CerbosAction.CREATE, sql_model, gql_type ) return typing.cast(T, create) @@ -235,16 +192,16 @@ async def update( # Fetch entity for update, if we have access to it filters = [sql_model.id == entity_id] - entities = await get_entities(sql_model, session, cerbos_client, principal, filters, []) # type: ignore + entities = await get_db_rows(sql_model, session, cerbos_client, principal, filters, []) # type: ignore if len(entities) != 1: raise Exception("Unauthorized: Cannot retrieve entity") entity = entities[0] - # Validate that user can update this entity. For now, this is redundant with get_entities() above, + # Validate that user can update this entity. For now, this is redundant with get_db_rows() above, # but it's possible we'll want "update" actions to require additional permissions in the future. attr = {"collection_id": entity.collection_id} resource = Resource(id=str(entity.id), kind=sql_model.__tablename__, attr=attr) - if not cerbos_client.is_allowed(CERBOS_ACTION_UPDATE, principal, resource): + if not cerbos_client.is_allowed(CerbosAction.UPDATE, principal, resource): raise Exception("Unauthorized: Cannot update entity") # Update DB @@ -256,7 +213,7 @@ async def update( return entity update.arguments = generate_strawberry_arguments( - CERBOS_ACTION_UPDATE, sql_model, gql_type + CerbosAction.UPDATE, sql_model, gql_type ) return typing.cast(T, update) @@ -269,13 +226,13 @@ def generate_strawberry_arguments( sql_columns = [column.name for column in sql_model.__table__.columns] # Always create an entity within a collection ID so need to specify it - if action == CERBOS_ACTION_CREATE: + if action == CerbosAction.CREATE: sql_columns.append("collection_id") gql_arguments = [] for sql_column in sql_columns: # Entity ID is autogenerated, so don't let user specify it unless updating a field - if action != CERBOS_ACTION_UPDATE and sql_column == "entity_id": + if action != CerbosAction.UPDATE and sql_column == "entity_id": continue # Get GQL field @@ -283,7 +240,7 @@ def generate_strawberry_arguments( if field: # When updating an entity, only entity ID is required is_optional_field = ( - action == CERBOS_ACTION_UPDATE and field.name != "entity_id" + action == CerbosAction.UPDATE and field.name != "entity_id" ) default = None if is_optional_field else strawberry.UNSET argument = StrawberryArgument( diff --git a/platformics/security/__init__.py b/platformics/security/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/platformics/security/authorization.py b/platformics/security/authorization.py new file mode 100644 index 00000000..f8c45cde --- /dev/null +++ b/platformics/security/authorization.py @@ -0,0 +1,38 @@ + +from platformics.thirdparty.cerbos_sqlalchemy.query import get_query +from cerbos.sdk.client import CerbosClient +import database.models as db +from cerbos.sdk.model import Principal, Resource, ResourceDesc +from enum import Enum +import typing + +E = typing.TypeVar("E", bound=typing.Union[db.Base, db.Entity]) + +class CerbosAction(str, Enum): + VIEW = "view" + CREATE = "create" + UPDATE = "update" + DELETE = "delete" + +def get_resource_query(principal: Principal, cerbos_client: CerbosClient, action: CerbosAction, model_cls: typing.Union[db.Entity, db.File]): + rd = ResourceDesc(model_cls.__tablename__) + plan = cerbos_client.plan_resources(action, principal, rd) + if model_cls == db.File: + attr_map = { + "request.resource.attr.owner_user_id": db.Entity.owner_user_id, + "request.resource.attr.collection_id": db.Entity.collection_id, + } + joins = [(db.Entity, model_cls.entity_id == db.Entity.id)], # type: ignore + else: + attr_map = { + "request.resource.attr.owner_user_id": model_cls.owner_user_id, + "request.resource.attr.collection_id": model_cls.collection_id, + } + joins = [] + query = get_query( + plan, + model_cls, # type: ignore + attr_map, + joins, + ) + return query \ No newline at end of file From c75d953c98afbbb9eedf7b57db54c81a3c3c0332 Mon Sep 17 00:00:00 2001 From: Jessica Gadling Date: Wed, 20 Sep 2023 13:23:34 -0700 Subject: [PATCH 4/8] Handle updating file status. --- entities/files/format_handlers.py | 4 ++-- platformics/api/core/gql_loaders.py | 2 +- platformics/security/authorization.py | 17 ++++++++--------- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/entities/files/format_handlers.py b/entities/files/format_handlers.py index 40eaef39..681a2d98 100644 --- a/entities/files/format_handlers.py +++ b/entities/files/format_handlers.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod +from abc import abstractmethod from mypy_boto3_s3.client import S3Client from Bio import SeqIO from io import StringIO @@ -32,9 +32,9 @@ def validate(cls, client: S3Client, bucket: str, file_path: str) -> int: def convert_to(cls, client: S3Client, bucket: str, file_path: str, format: dict) -> str: return "" + def get_validator(format: dict) -> type[FileFormatHandler]: if format["name"] == "fastq": return FastqHandler else: raise Exception("Unknown file format") - diff --git a/platformics/api/core/gql_loaders.py b/platformics/api/core/gql_loaders.py index c5dd8875..8ef1fa35 100644 --- a/platformics/api/core/gql_loaders.py +++ b/platformics/api/core/gql_loaders.py @@ -20,7 +20,7 @@ from strawberry.dataloader import DataLoader from platformics.security.authorization import get_resource_query, CerbosAction -E = typing.TypeVar("E", bound=db.Base) +E = typing.TypeVar("E", db.File, db.Entity) T = typing.TypeVar("T") diff --git a/platformics/security/authorization.py b/platformics/security/authorization.py index f8c45cde..79594ba8 100644 --- a/platformics/security/authorization.py +++ b/platformics/security/authorization.py @@ -5,8 +5,7 @@ from cerbos.sdk.model import Principal, Resource, ResourceDesc from enum import Enum import typing - -E = typing.TypeVar("E", bound=typing.Union[db.Base, db.Entity]) +from sqlalchemy.sql import Select class CerbosAction(str, Enum): VIEW = "view" @@ -14,7 +13,7 @@ class CerbosAction(str, Enum): UPDATE = "update" DELETE = "delete" -def get_resource_query(principal: Principal, cerbos_client: CerbosClient, action: CerbosAction, model_cls: typing.Union[db.Entity, db.File]): +def get_resource_query(principal: Principal, cerbos_client: CerbosClient, action: CerbosAction, model_cls: typing.Union[type[db.Entity], type[db.File]]) -> Select: rd = ResourceDesc(model_cls.__tablename__) plan = cerbos_client.plan_resources(action, principal, rd) if model_cls == db.File: @@ -25,14 +24,14 @@ def get_resource_query(principal: Principal, cerbos_client: CerbosClient, action joins = [(db.Entity, model_cls.entity_id == db.Entity.id)], # type: ignore else: attr_map = { - "request.resource.attr.owner_user_id": model_cls.owner_user_id, - "request.resource.attr.collection_id": model_cls.collection_id, + "request.resource.attr.owner_user_id": model_cls.owner_user_id, # type: ignore + "request.resource.attr.collection_id": model_cls.collection_id, # type: ignore } - joins = [] + joins = None query = get_query( plan, - model_cls, # type: ignore - attr_map, - joins, + model_cls, # type: ignore + attr_map, # type: ignore + joins, # type: ignore ) return query \ No newline at end of file From cfc3a443311eb844f9cba24137104c621311e2c2 Mon Sep 17 00:00:00 2001 From: Jessica Gadling Date: Wed, 20 Sep 2023 14:57:09 -0700 Subject: [PATCH 5/8] Formatting. --- platformics/api/core/deps.py | 5 +-- platformics/api/core/gql_loaders.py | 43 +++++--------------- platformics/database/connect.py | 7 +--- platformics/pyproject.toml | 58 +++++++++++++++++++++++++++ platformics/security/authorization.py | 24 ++++++----- 5 files changed, 87 insertions(+), 50 deletions(-) create mode 100644 platformics/pyproject.toml diff --git a/platformics/api/core/deps.py b/platformics/api/core/deps.py index b577909f..49e4804b 100644 --- a/platformics/api/core/deps.py +++ b/platformics/api/core/deps.py @@ -44,9 +44,7 @@ def get_cerbos_client(settings: APISettings = Depends(get_settings)) -> CerbosCl return CerbosClient(host=settings.CERBOS_URL) -def get_auth_principal( - request: Request, settings: APISettings = Depends(get_settings) -) -> typing.Optional[Principal]: +def get_auth_principal(request: Request, settings: APISettings = Depends(get_settings)) -> typing.Optional[Principal]: auth_header = request.headers.get("authorization") if auth_header: parts = auth_header.split() @@ -86,6 +84,7 @@ def require_auth_principal( raise Exception("Unauthorized") return principal + def get_s3_client( settings: APISettings = Depends(get_settings), ) -> S3Client: diff --git a/platformics/api/core/gql_loaders.py b/platformics/api/core/gql_loaders.py index 8ef1fa35..4f5b586a 100644 --- a/platformics/api/core/gql_loaders.py +++ b/platformics/api/core/gql_loaders.py @@ -6,13 +6,11 @@ import database.models as db import strawberry from cerbos.sdk.client import CerbosClient -from cerbos.sdk.model import Principal, Resource, ResourceDesc +from cerbos.sdk.model import Principal, Resource from fastapi import Depends -from platformics.api.core.deps import (get_cerbos_client, get_db_session, - require_auth_principal) +from platformics.api.core.deps import get_cerbos_client, get_db_session, require_auth_principal from platformics.api.core.strawberry_extensions import DependencyExtension from platformics.database.connect import AsyncDB -from platformics.thirdparty.cerbos_sqlalchemy.query import get_query from sqlalchemy import ColumnElement, ColumnExpressionArgument, tuple_ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import RelationshipProperty @@ -48,9 +46,7 @@ class EntityLoader: _loaders: dict[RelationshipProperty, DataLoader] - def __init__( - self, engine: AsyncDB, cerbos_client: CerbosClient, principal: Principal - ) -> None: + def __init__(self, engine: AsyncDB, cerbos_client: CerbosClient, principal: Principal) -> None: self._loaders = {} self.engine = engine self.cerbos_client = cerbos_client @@ -70,11 +66,7 @@ def loader_for(self, relationship: RelationshipProperty) -> DataLoader: async def load_fn(keys: list[Tuple]) -> typing.Sequence[Any]: if not relationship.local_remote_pairs: raise Exception("invalid relationship") - filters = [ - tuple_( - *[remote for _, remote in relationship.local_remote_pairs] - ).in_(keys) - ] + filters = [tuple_(*[remote for _, remote in relationship.local_remote_pairs]).in_(keys)] order_by: list[tuple[ColumnElement[Any], ...]] = [] if relationship.order_by: order_by = [relationship.order_by] @@ -93,11 +85,7 @@ def group_by_remote_key(row: Any) -> Tuple: if not relationship.local_remote_pairs: raise Exception("invalid relationship") return tuple( - [ - getattr(row, remote.key) - for _, remote in relationship.local_remote_pairs - if remote.key - ] + [getattr(row, remote.key) for _, remote in relationship.local_remote_pairs if remote.key] ) grouped_keys: Mapping[Tuple, list[Any]] = defaultdict(list) @@ -106,10 +94,7 @@ def group_by_remote_key(row: Any) -> Tuple: if relationship.uselist: return [grouped_keys[key] for key in keys] else: - return [ - grouped_keys[key][0] if grouped_keys[key] else None - for key in keys - ] + return [grouped_keys[key][0] if grouped_keys[key] else None for key in keys] self._loaders[relationship] = DataLoader(load_fn=load_fn) return self._loaders[relationship] @@ -173,9 +158,7 @@ async def create( return new_entity - create.arguments = generate_strawberry_arguments( - CerbosAction.CREATE, sql_model, gql_type - ) + create.arguments = generate_strawberry_arguments(CerbosAction.CREATE, sql_model, gql_type) return typing.cast(T, create) @@ -212,9 +195,7 @@ async def update( return entity - update.arguments = generate_strawberry_arguments( - CerbosAction.UPDATE, sql_model, gql_type - ) + update.arguments = generate_strawberry_arguments(CerbosAction.UPDATE, sql_model, gql_type) return typing.cast(T, update) @@ -239,13 +220,9 @@ def generate_strawberry_arguments( field = gql_type.__strawberry_definition__.get_field(sql_column) # type: ignore if field: # When updating an entity, only entity ID is required - is_optional_field = ( - action == CerbosAction.UPDATE and field.name != "entity_id" - ) + is_optional_field = action == CerbosAction.UPDATE and field.name != "entity_id" default = None if is_optional_field else strawberry.UNSET - argument = StrawberryArgument( - field.name, field.graphql_name, field.type_annotation, default=default - ) + argument = StrawberryArgument(field.name, field.graphql_name, field.type_annotation, default=default) gql_arguments.append(argument) return gql_arguments diff --git a/platformics/database/connect.py b/platformics/database/connect.py index 639281aa..d147172d 100644 --- a/platformics/database/connect.py +++ b/platformics/database/connect.py @@ -1,8 +1,7 @@ from typing import Any, Optional from sqlalchemy.engine import Engine, create_engine -from sqlalchemy.ext.asyncio import (AsyncEngine, AsyncSession, - async_sessionmaker, create_async_engine) +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.orm import Session, sessionmaker @@ -41,9 +40,7 @@ def session(self) -> sessionmaker[Session]: def init_async_db(db_uri: str, **kwargs: dict[str, Any]) -> AsyncDB: - engine = create_async_engine( - db_uri, echo=False, pool_size=5, max_overflow=5, future=True, **kwargs - ) + engine = create_async_engine(db_uri, echo=False, pool_size=5, max_overflow=5, future=True, **kwargs) return AsyncDB(engine) diff --git a/platformics/pyproject.toml b/platformics/pyproject.toml new file mode 100644 index 00000000..3bdf8bd8 --- /dev/null +++ b/platformics/pyproject.toml @@ -0,0 +1,58 @@ +[tool.black] +line-length = 120 + +[tool.mypy] +explicit_package_bases = true +ignore_missing_imports = true +disallow_untyped_defs = true +exclude = [ + 'gql_schema\.py$', + '^platformics/thirdparty', +] + +[tool.ruff] +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +select = ["E", "F"] +ignore = [] + +# Allow autofix for all enabled rules (when `--fix`) is provided. +fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] +unfixable = [] + +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", +] + +# Same as Black. +line-length = 120 + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +# Assume Python 3.11. +target-version = "py311" + +[tool.ruff.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 diff --git a/platformics/security/authorization.py b/platformics/security/authorization.py index 79594ba8..7caff399 100644 --- a/platformics/security/authorization.py +++ b/platformics/security/authorization.py @@ -1,19 +1,25 @@ - from platformics.thirdparty.cerbos_sqlalchemy.query import get_query from cerbos.sdk.client import CerbosClient import database.models as db -from cerbos.sdk.model import Principal, Resource, ResourceDesc +from cerbos.sdk.model import Principal, ResourceDesc from enum import Enum import typing from sqlalchemy.sql import Select + class CerbosAction(str, Enum): VIEW = "view" CREATE = "create" UPDATE = "update" DELETE = "delete" -def get_resource_query(principal: Principal, cerbos_client: CerbosClient, action: CerbosAction, model_cls: typing.Union[type[db.Entity], type[db.File]]) -> Select: + +def get_resource_query( + principal: Principal, + cerbos_client: CerbosClient, + action: CerbosAction, + model_cls: typing.Union[type[db.Entity], type[db.File]], +) -> Select: rd = ResourceDesc(model_cls.__tablename__) plan = cerbos_client.plan_resources(action, principal, rd) if model_cls == db.File: @@ -21,17 +27,17 @@ def get_resource_query(principal: Principal, cerbos_client: CerbosClient, action "request.resource.attr.owner_user_id": db.Entity.owner_user_id, "request.resource.attr.collection_id": db.Entity.collection_id, } - joins = [(db.Entity, model_cls.entity_id == db.Entity.id)], # type: ignore + joins = ([(db.Entity, model_cls.entity_id == db.Entity.id)],) # type: ignore else: attr_map = { - "request.resource.attr.owner_user_id": model_cls.owner_user_id, # type: ignore - "request.resource.attr.collection_id": model_cls.collection_id, # type: ignore + "request.resource.attr.owner_user_id": model_cls.owner_user_id, # type: ignore + "request.resource.attr.collection_id": model_cls.collection_id, # type: ignore } joins = None query = get_query( plan, - model_cls, # type: ignore - attr_map, # type: ignore + model_cls, # type: ignore + attr_map, # type: ignore joins, # type: ignore ) - return query \ No newline at end of file + return query From 4436811767aa3eddd95a3792d46ea81d4d25d9ae Mon Sep 17 00:00:00 2001 From: Jessica Gadling Date: Wed, 20 Sep 2023 15:39:35 -0700 Subject: [PATCH 6/8] Make validation work. --- entities/api/files.py | 3 +-- entities/files/format_handlers.py | 6 +++--- platformics/security/authorization.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/entities/api/files.py b/entities/api/files.py index 2d6058e2..a438a47f 100644 --- a/entities/api/files.py +++ b/entities/api/files.py @@ -49,7 +49,6 @@ async def mark_upload_complete( cerbos_client: CerbosClient = Depends(get_cerbos_client), session: AsyncSession = Depends(get_db_session, use_cache=False), s3_client: S3Client = Depends(get_s3_client), - **kwargs: typing.Any, ) -> db.File: query = get_resource_query(principal, cerbos_client, CerbosAction.UPDATE, db.File) query = query.filter(db.File.id == file_id) @@ -59,7 +58,7 @@ async def mark_upload_complete( validator = get_validator(file.file_format) try: - file_size = validator.validate(s3_client, file.namespace, file.path) + file_size = validator.validate(s3_client, file.namespace, file.path.lstrip("/")) except: # noqa raise Exception("VALIDATION FAILURE!!") diff --git a/entities/files/format_handlers.py b/entities/files/format_handlers.py index 681a2d98..e6490a58 100644 --- a/entities/files/format_handlers.py +++ b/entities/files/format_handlers.py @@ -23,7 +23,7 @@ def validate(cls, client: S3Client, bucket: str, file_path: str) -> int: # Overly simplistic validator for fastq filees checks whether the first 1mb of a file are a valid fastq data = client.get_object(Bucket=bucket, Key=file_path, Range="bytes=0-1000000")["Body"].read() records = 0 - for _ in SeqIO.parse(StringIO(str(data)), "fasta"): + for _ in SeqIO.parse(StringIO(data.decode("ascii")), "fastq"): records += 1 assert records > 0 return client.head_object(Bucket=bucket, Key=file_path)["ContentLength"] @@ -33,8 +33,8 @@ def convert_to(cls, client: S3Client, bucket: str, file_path: str, format: dict) return "" -def get_validator(format: dict) -> type[FileFormatHandler]: - if format["name"] == "fastq": +def get_validator(format: str) -> type[FileFormatHandler]: + if format == "fastq": return FastqHandler else: raise Exception("Unknown file format") diff --git a/platformics/security/authorization.py b/platformics/security/authorization.py index 7caff399..1697aca5 100644 --- a/platformics/security/authorization.py +++ b/platformics/security/authorization.py @@ -27,7 +27,7 @@ def get_resource_query( "request.resource.attr.owner_user_id": db.Entity.owner_user_id, "request.resource.attr.collection_id": db.Entity.collection_id, } - joins = ([(db.Entity, model_cls.entity_id == db.Entity.id)],) # type: ignore + joins = ([(db.Entity, model_cls.entity_id == db.Entity.id)]) # type: ignore else: attr_map = { "request.resource.attr.owner_user_id": model_cls.owner_user_id, # type: ignore From ff88ae6a306ef94ba601e7d2d0c47d7e216d0bdf Mon Sep 17 00:00:00 2001 From: Jessica Gadling Date: Wed, 20 Sep 2023 16:46:23 -0700 Subject: [PATCH 7/8] Add tests! --- entities/api/conftest.py | 27 +++++- entities/api/files.py | 8 +- entities/api/tests/test_file_writes.py | 82 ++++++++++++++++ entities/poetry.lock | 126 ++++++++++++++++++++++++- entities/pyproject.toml | 1 + entities/test_infra/factories.py | 2 +- 6 files changed, 239 insertions(+), 7 deletions(-) create mode 100644 entities/api/tests/test_file_writes.py diff --git a/entities/api/conftest.py b/entities/api/conftest.py index 8ef50fec..5fd83b7a 100644 --- a/entities/api/conftest.py +++ b/entities/api/conftest.py @@ -2,6 +2,7 @@ import typing from typing import Optional +import boto3 import pytest_asyncio from cerbos.sdk.model import Principal from platformics.database.connect import AsyncDB @@ -9,8 +10,16 @@ from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession from starlette.requests import Request +from moto import mock_s3 +from mypy_boto3_s3.client import S3Client -from platformics.api.core.deps import get_auth_principal, get_db_session, get_engine, require_auth_principal +from platformics.api.core.deps import ( + get_auth_principal, + get_db_session, + get_engine, + require_auth_principal, + get_s3_client, +) from api.main import get_app @@ -43,6 +52,21 @@ async def query( return result.json() +@pytest_asyncio.fixture() +async def moto_client() -> S3Client: + mocks3 = mock_s3() + mocks3.start() + res = boto3.resource("s3") + res.create_bucket(Bucket="local-bucket") + res.create_bucket(Bucket="remote-bucket") + yield boto3.client("s3") + mocks3.stop() + + +async def patched_s3_client() -> S3Client: + yield boto3.client("s3") + + @pytest_asyncio.fixture() async def gql_client(http_client: AsyncClient) -> GQLTestClient: client = GQLTestClient(http_client) @@ -81,6 +105,7 @@ async def patched_session() -> typing.AsyncGenerator[AsyncSession, None]: api.dependency_overrides[get_db_session] = patched_session api.dependency_overrides[require_auth_principal] = patched_authprincipal api.dependency_overrides[get_auth_principal] = patched_authprincipal + api.dependency_overrides[get_s3_client] = patched_s3_client return api diff --git a/entities/api/files.py b/entities/api/files.py index a438a47f..fc0fd744 100644 --- a/entities/api/files.py +++ b/entities/api/files.py @@ -60,9 +60,9 @@ async def mark_upload_complete( try: file_size = validator.validate(s3_client, file.namespace, file.path.lstrip("/")) except: # noqa - raise Exception("VALIDATION FAILURE!!") - - file.status = db.FileStatus.SUCCESS - file.size = file_size + file.status = db.FileStatus.FAILED + else: + file.status = db.FileStatus.SUCCESS + file.size = file_size return file diff --git a/entities/api/tests/test_file_writes.py b/entities/api/tests/test_file_writes.py new file mode 100644 index 00000000..31943d19 --- /dev/null +++ b/entities/api/tests/test_file_writes.py @@ -0,0 +1,82 @@ +import os +import pytest +from api.conftest import GQLTestClient +from platformics.database.connect import SyncDB +from test_infra import factories as fa +from mypy_boto3_s3.client import S3Client +from database.models import File +import sqlalchemy as sa + + +# Test that we can mark a file upload as complete +@pytest.mark.asyncio +async def test_file_validation( + sync_db: SyncDB, + gql_client: GQLTestClient, + moto_client: S3Client, +) -> None: + user1_id = 12345 + project1_id = 123 + + # Create mock data + with sync_db.session() as session: + fa.SessionStorage.set_session(session) + fa.SequencingReadFactory.create(owner_user_id=user1_id, collection_id=project1_id) + fa.FileFactory.update_file_ids() + session.commit() + file = session.execute(sa.select(File)).scalars().one() + + valid_fastq_file = "test_infra/fixtures/test1.fastq" + moto_client.put_object(Bucket=file.namespace, Key=file.path.lstrip("/"), Body=open(valid_fastq_file, "rb")) + + # Mark upload complete + query = f""" + mutation MyMutation {{ + markUploadComplete(fileId: "{file.id}") {{ + id + namespace + size + status + }} + }} + """ + res = await gql_client.query(query, member_projects=[project1_id]) + fileinfo = res["data"]["markUploadComplete"] + assert fileinfo["status"] == "SUCCESS" + assert fileinfo["size"] == os.stat(valid_fastq_file).st_size + + +# Test that invalid fastq's don't work +@pytest.mark.asyncio +async def test_invalid_fastq( + sync_db: SyncDB, + gql_client: GQLTestClient, + moto_client: S3Client, +) -> None: + user1_id = 12345 + project1_id = 123 + + # Create mock data + with sync_db.session() as session: + fa.SessionStorage.set_session(session) + fa.SequencingReadFactory.create(owner_user_id=user1_id, collection_id=project1_id) + fa.FileFactory.update_file_ids() + session.commit() + file = session.execute(sa.select(File)).scalars().one() + + moto_client.put_object(Bucket=file.namespace, Key=file.path.lstrip("/"), Body="this is not a fastq file") + + # Mark upload complete + query = f""" + mutation MyMutation {{ + markUploadComplete(fileId: "{file.id}") {{ + id + namespace + size + status + }} + }} + """ + res = await gql_client.query(query, member_projects=[project1_id]) + fileinfo = res["data"]["markUploadComplete"] + assert fileinfo["status"] == "FAILED" diff --git a/entities/poetry.lock b/entities/poetry.lock index 06b2f4bb..616cdf88 100644 --- a/entities/poetry.lock +++ b/entities/poetry.lock @@ -1377,6 +1377,23 @@ pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib" plugins = ["setuptools"] requirements-deprecated-finder = ["pip-api", "pipreqs"] +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + [[package]] name = "jmespath" version = "1.0.1" @@ -1620,6 +1637,54 @@ files = [ [package.dependencies] psutil = {version = ">=4.0.0", markers = "sys_platform != \"cygwin\""} +[[package]] +name = "moto" +version = "4.2.3" +description = "" +optional = false +python-versions = ">=3.7" +files = [ + {file = "moto-4.2.3-py2.py3-none-any.whl", hash = "sha256:2e934d834729b274382055e097b166127db829ab4fae00bb08c031c108391a2c"}, + {file = "moto-4.2.3.tar.gz", hash = "sha256:4caab0145d557d102fe79d0ce3b73d6bf1d916d29ad03c14da15f7da66429cdb"}, +] + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.12.201" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +python-dateutil = ">=2.1,<3.0.0" +requests = ">=2.5" +responses = ">=0.13.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "py-partiql-parser (==0.3.7)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +apigateway = ["PyYAML (>=5.1)", "ecdsa (!=0.15)", "openapi-spec-validator (>=0.2.8)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +apigatewayv2 = ["PyYAML (>=5.1)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=3.0.0)"] +batch = ["docker (>=3.0.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "py-partiql-parser (==0.3.7)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +cognitoidp = ["ecdsa (!=0.15)", "python-jose[cryptography] (>=3.1.0,<4.0.0)"] +ds = ["sshpubkeys (>=3.1.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.3.7)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.3.7)"] +ebs = ["sshpubkeys (>=3.1.0)"] +ec2 = ["sshpubkeys (>=3.1.0)"] +efs = ["sshpubkeys (>=3.1.0)"] +eks = ["sshpubkeys (>=3.1.0)"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "py-partiql-parser (==0.3.7)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "sshpubkeys (>=3.1.0)"] +route53resolver = ["sshpubkeys (>=3.1.0)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.3.7)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.3.7)"] +server = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "ecdsa (!=0.15)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.2.8)", "py-partiql-parser (==0.3.7)", "pyparsing (>=3.0.7)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "setuptools", "sshpubkeys (>=3.1.0)"] +ssm = ["PyYAML (>=5.1)"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + [[package]] name = "mypy" version = "1.5.1" @@ -2271,6 +2336,26 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" +[[package]] +name = "responses" +version = "0.23.3" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "responses-0.23.3-py3-none-any.whl", hash = "sha256:e6fbcf5d82172fecc0aa1860fd91e58cbfd96cee5e96da5b63fa6eb3caa10dd3"}, + {file = "responses-0.23.3.tar.gz", hash = "sha256:205029e1cb334c21cb4ec64fc7599be48b859a0fd381a42443cdd600bfe8b16a"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +types-PyYAML = "*" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] + [[package]] name = "rich" version = "13.4.2" @@ -2596,6 +2681,17 @@ files = [ {file = "types_awscrt-0.19.1.tar.gz", hash = "sha256:61833aa140e724a9098025610f4b8cde3dcf65b842631d7447378f9f5db4e1fd"}, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.11" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.11.tar.gz", hash = "sha256:7d340b19ca28cddfdba438ee638cd4084bde213e501a3978738543e27094775b"}, + {file = "types_PyYAML-6.0.12.11-py3-none-any.whl", hash = "sha256:a461508f3096d1d5810ec5ab95d7eeecb651f3a15b71959999988942063bf01d"}, +] + [[package]] name = "types-requests" version = "2.31.0.2" @@ -2714,6 +2810,23 @@ h11 = ">=0.8" [package.extras] standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] +[[package]] +name = "werkzeug" +version = "2.3.7" +description = "The comprehensive WSGI web application library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "werkzeug-2.3.7-py3-none-any.whl", hash = "sha256:effc12dba7f3bd72e605ce49807bbe692bd729c3bb122a3b91747a6ae77df528"}, + {file = "werkzeug-2.3.7.tar.gz", hash = "sha256:2b8c0e447b4b9dbcc85dd97b6eeb4dcbaf6c8b6c3be0bd654e25553e0a2157d8"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog (>=2.3)"] + [[package]] name = "wrapt" version = "1.15.0" @@ -2798,6 +2911,17 @@ files = [ {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, ] +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.4" +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] + [[package]] name = "zipp" version = "3.16.2" @@ -2816,4 +2940,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "d0a11dfd0392919c8832f53b8e7c2d781cef52332aebfa5b175ab03301c8528e" +content-hash = "e0d79ef3225a3c907de02b84271e67ab1ce44da94b924d3e83703ec7a35173ff" diff --git a/entities/pyproject.toml b/entities/pyproject.toml index 85a98c88..a441afef 100644 --- a/entities/pyproject.toml +++ b/entities/pyproject.toml @@ -39,6 +39,7 @@ boto3 = "^1.28.43" boto3-stubs = {extras = ["s3"], version = "^1.28.50"} faker-enum = "^0.0.2" biopython = "^1.81" +moto = "^4.2.3" [build-system] requires = ["poetry-core"] diff --git a/entities/test_infra/factories.py b/entities/test_infra/factories.py index 8c56b7e9..9795281b 100644 --- a/entities/test_infra/factories.py +++ b/entities/test_infra/factories.py @@ -110,7 +110,7 @@ class Meta: # sequence = factory.Faker('dna', length=100) protocol = fuzzy.FuzzyChoice(["TARGETED", "MNGS", "MSSPE"]) - sequencing_read_file = factory.RelatedFactory( + sequence_file = factory.RelatedFactory( FileFactory, factory_related_name="entity", entity_field_name="sequence_file", From 32575c6716e5dc76c44e79f2ea2e03f757448d45 Mon Sep 17 00:00:00 2001 From: Jessica Gadling Date: Thu, 21 Sep 2023 10:21:42 -0700 Subject: [PATCH 8/8] Fix broken stuff. --- entities/api/conftest.py | 4 ++-- platformics/security/authorization.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/entities/api/conftest.py b/entities/api/conftest.py index 5fd83b7a..4d8f5340 100644 --- a/entities/api/conftest.py +++ b/entities/api/conftest.py @@ -53,7 +53,7 @@ async def query( @pytest_asyncio.fixture() -async def moto_client() -> S3Client: +async def moto_client() -> typing.AsyncGenerator[S3Client, None]: mocks3 = mock_s3() mocks3.start() res = boto3.resource("s3") @@ -63,7 +63,7 @@ async def moto_client() -> S3Client: mocks3.stop() -async def patched_s3_client() -> S3Client: +async def patched_s3_client() -> typing.AsyncGenerator[S3Client, None]: yield boto3.client("s3") diff --git a/platformics/security/authorization.py b/platformics/security/authorization.py index 1697aca5..114d4116 100644 --- a/platformics/security/authorization.py +++ b/platformics/security/authorization.py @@ -33,7 +33,7 @@ def get_resource_query( "request.resource.attr.owner_user_id": model_cls.owner_user_id, # type: ignore "request.resource.attr.collection_id": model_cls.collection_id, # type: ignore } - joins = None + joins = [] query = get_query( plan, model_cls, # type: ignore