From 934aabdd6745b3e8484622373c6927ae2214704e Mon Sep 17 00:00:00 2001 From: Chase Yakaboski Date: Fri, 28 Jul 2023 13:58:35 -0400 Subject: [PATCH] All five algorithms work as expected. --- chp_api/chp_api/settings.py | 3 + chp_api/gennifer/models.py | 5 +- chp_api/gennifer/serializers.py | 19 ++++- chp_api/gennifer/tasks.py | 6 ++ chp_api/gennifer/urls.py | 1 + chp_api/gennifer/views.py | 137 +++++++++++++++++++++++++------- compose.chp-api.yaml | 4 +- compose.gennifer.yaml | 54 +++++++++++++ deployment-script | 2 +- nginx/default.conf | 18 ++++- 10 files changed, 211 insertions(+), 38 deletions(-) diff --git a/chp_api/chp_api/settings.py b/chp_api/chp_api/settings.py index da9d796..3ae79b7 100644 --- a/chp_api/chp_api/settings.py +++ b/chp_api/chp_api/settings.py @@ -247,4 +247,7 @@ GENNIFER_ALGORITHM_URLS = [ "http://pidc:5000", "http://grisli:5000", + "http://genie3:5000", + "http://grnboost2:5000", + "http://bkb-grn:5000", ] diff --git a/chp_api/gennifer/models.py b/chp_api/gennifer/models.py index a025099..d506c26 100644 --- a/chp_api/gennifer/models.py +++ b/chp_api/gennifer/models.py @@ -37,7 +37,7 @@ class AlgorithmInstance(models.Model): def __str__(self): if self.hyperparameters: - hypers = tuple([f'{h}' for h in self.hyperparameters]) + hypers = tuple([f'{h}' for h in self.hyperparameters.all()]) else: hypers = '()' return f'{self.algorithm.name}{hypers}' @@ -81,7 +81,7 @@ def get_value(self): return self.hyperparameter.get_type()(self.value_str) def __str__(self): - return f'{self.hyperparameter.name}={self.value}' + return f'{self.hyperparameter.name}={self.value_str}' class Dataset(models.Model): title = models.CharField(max_length=128) @@ -133,6 +133,7 @@ class Meta: def __str__(self): return self.name + class Task(models.Model): algorithm_instance = models.ForeignKey(AlgorithmInstance, on_delete=models.CASCADE, related_name='tasks') user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True, related_name='tasks') diff --git a/chp_api/gennifer/serializers.py b/chp_api/gennifer/serializers.py index d310202..1e5e745 100644 --- a/chp_api/gennifer/serializers.py +++ b/chp_api/gennifer/serializers.py @@ -1,3 +1,4 @@ +from collections import defaultdict from rest_framework import serializers from .models import ( @@ -26,10 +27,24 @@ class Meta: read_only_fields = ['pk', 'title', 'doi', 'description'] class StudySerializer(serializers.ModelSerializer): + task_status = serializers.SerializerMethodField('get_task_status') + + def get_task_status(self, study): + status = defaultdict(int) + for task in study.tasks.all(): + status[task.status] += 1 + status = dict(status) + status = sorted([f'{count} {state}'.title() for state, count in status.items()]) + if len(status) == 0: + return '' + elif len(status) == 1: + return status[0] + return ' and '.join([', '.join(status[:-1]), status[-1]]) + class Meta: model = Study - fields = ['pk', 'name', 'status', 'description', 'timestamp', 'user', 'tasks'] - read_only_fields = ['pk', 'status'] + fields = ['pk', 'name', 'status', 'task_status', 'description', 'timestamp', 'user', 'tasks'] + read_only_fields = ['pk', 'status', 'task_status'] class TaskSerializer(serializers.ModelSerializer): diff --git a/chp_api/gennifer/tasks.py b/chp_api/gennifer/tasks.py index 560b143..25f3b4d 100644 --- a/chp_api/gennifer/tasks.py +++ b/chp_api/gennifer/tasks.py @@ -73,12 +73,18 @@ def save_inference_task(task, status, failed=False): except TypeError: gene1_name = 'Not found in SRI Node Normalizer.' gene1_chp_preferred_curie = None + except KeyError: + _, gene1_name = res[gene1]["id"]["identifier"].split(':') + gene1_chp_preferred_curie = get_chp_preferred_curie(res[gene1]) try: gene2_name = res[gene2]["id"]["label"] gene2_chp_preferred_curie = get_chp_preferred_curie(res[gene2]) except TypeError: gene2_name = 'Not found in SRI Node Normalizer.' gene2_chp_preferred_curie = None + except KeyError: + _, gene2_name = res[gene2]["id"]["identifier"].split(':') + gene2_chp_preferred_curie = get_chp_preferred_curie(res[gene2]) gene1_obj, created = Gene.objects.get_or_create( name=gene1_name, curie=gene1, diff --git a/chp_api/gennifer/urls.py b/chp_api/gennifer/urls.py index 4d9671e..3e380c0 100644 --- a/chp_api/gennifer/urls.py +++ b/chp_api/gennifer/urls.py @@ -20,4 +20,5 @@ path('', include(router.urls)), path('run/', views.run.as_view()), path('graph/', views.CytoscapeView.as_view()), + path('download_study/', views.StudyDownloadView.as_view()) ] diff --git a/chp_api/gennifer/views.py b/chp_api/gennifer/views.py index a795507..64f7fd9 100644 --- a/chp_api/gennifer/views.py +++ b/chp_api/gennifer/views.py @@ -136,12 +136,23 @@ class HyperparameterInstanceViewSet(viewsets.ModelViewSet): class GeneViewSet(viewsets.ModelViewSet): serializer_class = GeneSerializer - queryset = Gene.objects.all() + #queryset = Gene.objects.all() permission_classes = [IsAuthenticated, IsAdminOrReadOnly]#, TokenHasReadWriteScope] - - -class CytoscapeView(APIView): - + + def get_queryset(self): + user = self.request.user + # Get user results + results = Result.objects.filter(user=user) + tf_genes = Gene.objects.filter(inference_result_tf__pk__in=results) + target_genes = Gene.objects.filter(inference_result_target__pk__in=results) + genes_union = tf_genes.union(target_genes) + print(len(genes_union)) + return genes_union + +class CytoscapeHandler: + def __init__(self, results): + self.results = results + def construct_node(self, gene_obj): if gene_obj.variant: name = f'{gene_obj.name}({gene_obj.variant})' @@ -164,7 +175,7 @@ def construct_node(self, gene_obj): def construct_edge(self, res, source_id, target_id): # Normalize edge weight based on the study - normalized_weight = (res.edge_weight - res.task.min_study_edge_weight) / (res.task.max_study_edge_weight - res.task.min_study_edge_weight) + normalized_weight = (res.edge_weight - res.task.min_task_edge_weight) / (res.task.max_task_edge_weight - res.task.min_task_edge_weight) directed = res.task.algorithm_instance.algorithm.directed edge_tuple = tuple(sorted([source_id, target_id])) edge = { @@ -202,14 +213,14 @@ def add(self, res, nodes, edges, processed_node_ids, processed_undirected_edges) pass return nodes, edges, processed_node_ids, processed_undirected_edges - def construct_cytoscape_data(self, results): + def construct_cytoscape_data(self): nodes = [] edges = [] processed_node_ids = set() processed_undirected_edges = set() elements = [] # Construct graph - for res in results: + for res in self.results: nodes, edges, processed_node_ids, processed_undirected_edges = self.add( res, nodes, @@ -223,21 +234,26 @@ def construct_cytoscape_data(self, results): "elements": elements } + +class CytoscapeView(APIView): + + def get(self, request): results = Result.objects.all() - cyto = self.construct_cytoscape_data(results) + cyto_handler = CytoscapeHandler(results) + cyto = cyto_handler.construct_cytoscape_data() return JsonResponse(cyto) def post(self, request): elements = [] gene_ids = request.data.get("gene_ids", None) - task_ids = request.data.get("task_ids", None) + study_ids = request.data.get("study_ids", None) algorithm_ids = request.data.get("algorithm_ids", None) dataset_ids = request.data.get("dataset_ids", None) cached_inference_result_ids = request.data.get("cached_results", None) if not (study_ids and gene_ids) and not (algorithm_ids and dataset_ids and gene_ids): - return JsonResponse({"elements": elements}) + return JsonResponse({"elements": elements, "result_ids": []}) # Create Filter filters = [] @@ -249,9 +265,11 @@ def post(self, request): ] ) if study_ids: - filters.append({"field": 'task__pk', "operator": 'in', "value": study_ids}) + tasks = Task.objects.filter(study__pk__in=study_ids) + task_ids = [task.pk for task in tasks] + filters.append({"field": 'task__pk', "operator": 'in', "value": task_ids}) if algorithm_ids: - filters.append({"field": 'task__algorithm_instance__algorithm__pk', "operator": 'in', "value": study_ids}) + filters.append({"field": 'task__algorithm_instance__algorithm__pk', "operator": 'in', "value": algorithm_ids}) if dataset_ids: filters.append({"field": 'task__dataset__zenodo_id', "operator": 'in', "value": dataset_ids}) @@ -267,30 +285,89 @@ def post(self, request): results = Result.objects.filter(query) if len(results) == 0: - return JsonResponse({"elements": elements}) + return JsonResponse({"elements": elements, "result_ids": []}) # Exclude results that have already been sent to user if cached_inference_result_ids: - logs.append('filtering') results = results.exclude(pk__in=cached_inference_result_ids) - nodes = [] - edges = [] - processed_node_ids = set() - processed_undirected_edges = set() - for res in results: - nodes, edges, processed_node_ids, processed_undirected_edges = self.add( - res, - nodes, - edges, - processed_node_ids, - processed_undirected_edges, - ) - elements.extend(nodes) - elements.extend(edges) - return JsonResponse({"elements": elements}) + # Capture result_ids + result_ids = [res.pk for res in results] + + # Initialize Cytoscape Handler + cyto_handler = CytoscapeHandler(results) + elements_dict = cyto_handler.construct_cytoscape_data() + + elements_dict["result_ids"] = result_ids + return JsonResponse(elements_dict) +class StudyDownloadView(APIView): + permission_classes = [IsAuthenticated] + def get(self, request, study_id=None): + if not study_id: + return JsonResponse({"detail": 'No study ID was passed.'}) + # Set response + response = { + "study_id": study_id, + } + # Get study + try: + study = Study.objects.get(pk=study_id, user=request.user) + except ObjectDoesNotExist: + response["error"] = 'The study does not exist for request user.' + return JsonResponse(response) + + response["description"] = study.description + response["status"] = study.status + response["tasks"] = [] + + # Get tasks assocaited with study + tasks = Task.objects.filter(study=study) + + # Collect task information + for task in tasks: + task_json = {} + # Collect task information + task_json["max_task_edge_weight"] = task.max_task_edge_weight + task_json["min_task_edge_weight"] = task.min_task_edge_weight + task_json["avg_task_edge_weight"] = task.avg_task_edge_weight + task_json["std_task_edge_weight"] = task.std_task_edge_weight + task_json["status"] = task.status + # Collect algo hyperparameters + hyper_instance_objs = task.algorithm_instance.hyperparameters.all() + hypers = {} + for hyper in hyper_instance_objs: + hypers[hyper.hyperparameter.name] = { + "value": hyper.value_str, + "info": hyper.hyperparameter.info + } + # Collect Algorithm instance information + task_json["algorithm"] = { + "name": task.algorithm_instance.algorithm.name, + "description": task.algorithm_instance.algorithm.description, + "edge_weight_description": task.algorithm_instance.algorithm.edge_weight_description, + "edge_weight_type": task.algorithm_instance.algorithm.edge_weight_type, + "directed": task.algorithm_instance.algorithm.directed, + "hyperparameters": hypers + } + # Collect Dataset information + task_json["dataset"] = { + "title": task.dataset.title, + "zenodo_id": task.dataset.zenodo_id, + "description": task.dataset.description, + } + # Build cytoscape graph + if task.status == 'SUCCESS': + results = Result.objects.filter(task=task) + cyto_handler = CytoscapeHandler(results) + task_json["graph"] = cyto_handler.construct_cytoscape_data() + else: + task_json["graph"] = None + response["tasks"].append(task_json) + + return JsonResponse(response) + class run(APIView): permission_classes = [IsAuthenticated] diff --git a/compose.chp-api.yaml b/compose.chp-api.yaml index 2e9a63a..be95e4d 100644 --- a/compose.chp-api.yaml +++ b/compose.chp-api.yaml @@ -11,8 +11,8 @@ services: - DJANGO_SERVER_ADDR=api:8000 - STATIC_SERVER_ADDR=static-fs:8080 - FLOWER_DASHBOARD_ADDR=dashboard:5556 - #- NEXTJS_SERVER_ADDR=chatgpt:3000 - - NEXTJS_SERVER_ADDR=api:8000 + - NEXTJS_SERVER_ADDR=frontend:3000 + #- NEXTJS_SERVER_ADDR=api:8000 ports: - "80:80" depends_on: diff --git a/compose.gennifer.yaml b/compose.gennifer.yaml index ffa7a9c..6dc86c8 100644 --- a/compose.gennifer.yaml +++ b/compose.gennifer.yaml @@ -1,6 +1,22 @@ version: '3.8' services: + frontend: + build: + context: ./gennifer/frontend + dockerfile: Dockerfile + ports: + - 3000:3000 + environment: + - NEXTAUTH_SECRET=X/NTPIqf088gXiYFi7WF0iH3NRJRPE3nZ0oOkRXf5es= + - NEXTAUTH_URL=https://chp.thayer.dartmouth.edu + - NEXTAUTH_URL_INTERNAL=http://127.0.0.1:3000 + - CREDENTIALS_URL=https://chp.thayer.dartmouth.edu/o/token/ + - GENNIFER_CLIENT_ID=jHM4ETk5wi2WUVPElpMFJtZqwY2oBKHVMmTsY9ry + - GENNIFER_CLIENT_SECRET=hY0XfS8YLGMojWuvUOPga4sJpEO9isltF7Xk7wXjyFHwWmxkifRXcPbnmhUM0oVO4Zlz349jbtBePIlaafkWubReqEBCoIcCzaZLa2a9pIlq55yow2TBvMDHnImrXvig + - GENNIFER_USER_DETAILS_URL=https://chp.thayer.dartmouth.edu/users/me/ + - GENNIFER_BASE_URL=https://chp.thayer.dartmouth.edu/gennifer/api/ + - NEXT_PUBLIC_GENNIFER_BASE_URL=https://chp.thayer.dartmouth.edu/gennifer/api/ pidc: build: @@ -128,6 +144,44 @@ services: - grnboost2 - redis + bkb-grn: + build: + context: ./gennifer/bkb-grn + dockerfile: Dockerfile + restart: always + user: gennifer_user + ports: + - 5008:5000 + secrets: + - gennifer_key + environment: + - PYTHONUNBUFFERED=1 + - SECRET_KEY_FILE=/run/secrets/gennifer_key + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + #command: gunicorn -c gunicorn.config.py -b 0.0.0.0:5000 'grnboost2:create_app()' + command: flask --app bkb_grn run --debug --host 0.0.0.0 + depends_on: + - redis + + worker-bkb-grn: + build: + context: ./gennifer/bkb-grn + dockerfile: Dockerfile + secrets: + - gurobi_lic + command: celery --app bkb_grn.tasks.celery worker -Q bkb_grn --loglevel=info + environment: + - PYTHONUNBUFFERED=1 + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + - GRB_LICENSE_FILE=/run/secrets/gurobi_lic + depends_on: + - bkb-grn + - redis + secrets: gennifer_key: file: secrets/gennifer/secret_key.txt + gurobi_lic: + file: secrets/gennifer/gurobi.lic diff --git a/deployment-script b/deployment-script index c077fa1..ab3de33 100755 --- a/deployment-script +++ b/deployment-script @@ -20,4 +20,4 @@ docker compose -f compose.chp-api.yaml run api python3 manage.py runscript algor docker compose -f compose.chp-api.yaml run --user root api python3 manage.py collectstatic --noinput -echo "Check logs with: docker compose logs -f" +echo "Check logs with: docker compose -f compose.chp-api.yaml -f compose.gennifer.yaml logs -f" diff --git a/nginx/default.conf b/nginx/default.conf index 03c4dbc..ee86726 100644 --- a/nginx/default.conf +++ b/nginx/default.conf @@ -52,8 +52,24 @@ server { proxy_pass http://$NEXTJS_SERVER_ADDR; } - location /chat { + location /ui { proxy_pass http://$NEXTJS_SERVER_ADDR/; } + location /dashboard { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + + location /studies { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + + location /explore { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + + location /documentation { + proxy_pass http://$NEXTJS_SERVER_ADDR; + } + }